From 4c98c7ea3bf01f77206418814b2a56022bc5e2e6 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Sun, 1 Sep 2024 22:44:38 +0530 Subject: [PATCH 001/357] Fix RepositoriesServiceIT.testUpdateRepository from #15426 (#15559) Signed-off-by: Ashish Singh --- .../repositories/RepositoriesServiceIT.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java index 7022c6ae3b956..e4347fef744ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/RepositoriesServiceIT.java @@ -69,7 +69,13 @@ public void testUpdateRepository() { .next(); final Settings.Builder repoSettings = Settings.builder().put("location", randomRepoPath()); - createRepository(repositoryName, FsRepository.TYPE, repoSettings); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + FsRepository.TYPE, + true, + repoSettings + ); final GetRepositoriesResponse originalGetRepositoriesResponse = client.admin() .cluster() @@ -87,7 +93,13 @@ public void testUpdateRepository() { final boolean updated = randomBoolean(); final String updatedRepositoryType = updated ? "mock" : FsRepository.TYPE; - createRepository(repositoryName, updatedRepositoryType, repoSettings); + OpenSearchIntegTestCase.putRepositoryWithNoSettingOverrides( + client().admin().cluster(), + repositoryName, + updatedRepositoryType, + true, + repoSettings + ); final GetRepositoriesResponse updatedGetRepositoriesResponse = client.admin() .cluster() From ef472528af78cafbf3e62594141b8203e51782de Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 2 Sep 2024 09:07:11 +0530 Subject: [PATCH 002/357] Add support to skip pinned timestamp in remote translog garbage collector (#15416) Signed-off-by: Sachin Kale --- .../indices/create/RemoteSplitIndexIT.java | 18 +- .../index/translog/RemoteFsTranslog.java | 369 +++++++- .../opensearch/index/translog/Translog.java | 4 + .../transfer/TranslogTransferManager.java | 99 ++- .../transfer/TranslogTransferMetadata.java | 24 + .../indices/RemoteStoreSettings.java | 5 + .../index/translog/RemoteFsTranslogTests.java | 55 +- ...oteFsTranslogWithPinnedTimestampTests.java | 840 ++++++++++++++++++ .../TranslogTransferManagerTests.java | 121 +++ 9 files changed, 1441 insertions(+), 94 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java index 928c9e33e19cb..3038db4583cf4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteSplitIndexIT.java @@ -66,6 +66,7 @@ import org.opensearch.index.seqno.SeqNoStats; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase; @@ -109,13 +110,16 @@ public void cleanUp() throws Exception { assertAcked( client().admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get() ); - assertBusy(() -> { - try { - assertEquals(0, getFileCount(translogRepoPath)); - } catch (IOException e) { - fail(); - } - }, 30, TimeUnit.SECONDS); + // With pinned timestamp, we can have tlog files even after deletion. + if (RemoteStoreSettings.isPinnedTimestampsEnabled() == false) { + assertBusy(() -> { + try { + assertEquals(0, getFileCount(translogRepoPath)); + } catch (IOException e) { + fail(); + } + }, 30, TimeUnit.SECONDS); + } super.teardown(); } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index c533a31c310c7..f6afc4c508720 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -11,15 +11,19 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -30,6 +34,7 @@ import org.opensearch.index.translog.transfer.TranslogTransferMetadata; import org.opensearch.index.translog.transfer.listener.TranslogTransferListener; import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -39,11 +44,16 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -51,10 +61,13 @@ import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import java.util.stream.LongStream; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; /** * A Translog implementation which syncs local FS with a remote store @@ -68,6 +81,8 @@ public class RemoteFsTranslog extends Translog { private final Logger logger; private final TranslogTransferManager translogTransferManager; + // This tracker keeps track of local tranlog files that are uploaded to remote store. + // Once tlog files are deleted from local, we remove them from tracker even if the files still exist in remote translog. private final FileTransferTracker fileTransferTracker; private final BooleanSupplier startedPrimarySupplier; private final RemoteTranslogTransferTracker remoteTranslogTransferTracker; @@ -92,6 +107,9 @@ public class RemoteFsTranslog extends Translog { private final Semaphore syncPermit = new Semaphore(SYNC_PERMIT); private final AtomicBoolean pauseSync = new AtomicBoolean(false); private final boolean isTranslogMetadataEnabled; + private final Map metadataFilePinnedTimestampMap; + // For metadata files, with no min generation in the name, we cache generation data to avoid multiple reads. + private final Map> oldFormatMetadataFileGenerationMap; public RemoteFsTranslog( TranslogConfig config, @@ -105,6 +123,36 @@ public RemoteFsTranslog( BooleanSupplier startedPrimarySupplier, RemoteTranslogTransferTracker remoteTranslogTransferTracker, RemoteStoreSettings remoteStoreSettings + ) throws IOException { + this( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + blobStoreRepository, + threadPool, + startedPrimarySupplier, + remoteTranslogTransferTracker, + remoteStoreSettings, + 0 + ); + } + + public RemoteFsTranslog( + TranslogConfig config, + String translogUUID, + TranslogDeletionPolicy deletionPolicy, + LongSupplier globalCheckpointSupplier, + LongSupplier primaryTermSupplier, + LongConsumer persistedSequenceNumberConsumer, + BlobStoreRepository blobStoreRepository, + ThreadPool threadPool, + BooleanSupplier startedPrimarySupplier, + RemoteTranslogTransferTracker remoteTranslogTransferTracker, + RemoteStoreSettings remoteStoreSettings, + long timestamp ) throws IOException { super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer); logger = Loggers.getLogger(getClass(), shardId); @@ -112,6 +160,8 @@ public RemoteFsTranslog( this.remoteTranslogTransferTracker = remoteTranslogTransferTracker; fileTransferTracker = new FileTransferTracker(shardId, remoteTranslogTransferTracker); isTranslogMetadataEnabled = indexSettings().isTranslogMetadataEnabled(); + this.metadataFilePinnedTimestampMap = new HashMap<>(); + this.oldFormatMetadataFileGenerationMap = new HashMap<>(); this.translogTransferManager = buildTranslogTransferManager( blobStoreRepository, threadPool, @@ -123,7 +173,7 @@ public RemoteFsTranslog( isTranslogMetadataEnabled ); try { - download(translogTransferManager, location, logger, config.shouldSeedRemote()); + download(translogTransferManager, location, logger, config.shouldSeedRemote(), timestamp); Checkpoint checkpoint = readCheckpoint(location); logger.info("Downloaded data from remote translog till maxSeqNo = {}", checkpoint.maxSeqNo); this.readers.addAll(recoverFromFiles(checkpoint)); @@ -174,6 +224,32 @@ public static void download( Logger logger, boolean seedRemote, boolean isTranslogMetadataEnabled + ) throws IOException { + download( + repository, + shardId, + threadPool, + location, + pathStrategy, + remoteStoreSettings, + logger, + seedRemote, + isTranslogMetadataEnabled, + 0 + ); + } + + public static void download( + Repository repository, + ShardId shardId, + ThreadPool threadPool, + Path location, + RemoteStorePathStrategy pathStrategy, + RemoteStoreSettings remoteStoreSettings, + Logger logger, + boolean seedRemote, + boolean isTranslogMetadataEnabled, + long timestamp ) throws IOException { assert repository instanceof BlobStoreRepository : String.format( Locale.ROOT, @@ -195,11 +271,12 @@ public static void download( remoteStoreSettings, isTranslogMetadataEnabled ); - RemoteFsTranslog.download(translogTransferManager, location, logger, seedRemote); + RemoteFsTranslog.download(translogTransferManager, location, logger, seedRemote, timestamp); logger.trace(remoteTranslogTransferTracker.toString()); } - static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger, boolean seedRemote) + // Visible for testing + static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger, boolean seedRemote, long timestamp) throws IOException { /* In Primary to Primary relocation , there can be concurrent upload and download of translog. @@ -213,7 +290,7 @@ static void download(TranslogTransferManager translogTransferManager, Path locat boolean success = false; long startTimeMs = System.currentTimeMillis(); try { - downloadOnce(translogTransferManager, location, logger, seedRemote); + downloadOnce(translogTransferManager, location, logger, seedRemote, timestamp); success = true; return; } catch (FileNotFoundException | NoSuchFileException e) { @@ -227,13 +304,18 @@ static void download(TranslogTransferManager translogTransferManager, Path locat throw ex; } - private static void downloadOnce(TranslogTransferManager translogTransferManager, Path location, Logger logger, boolean seedRemote) - throws IOException { + private static void downloadOnce( + TranslogTransferManager translogTransferManager, + Path location, + Logger logger, + boolean seedRemote, + long timestamp + ) throws IOException { logger.debug("Downloading translog files from remote"); RemoteTranslogTransferTracker statsTracker = translogTransferManager.getRemoteTranslogTransferTracker(); long prevDownloadBytesSucceeded = statsTracker.getDownloadBytesSucceeded(); long prevDownloadTimeInMillis = statsTracker.getTotalDownloadTimeInMillis(); - TranslogTransferMetadata translogMetadata = translogTransferManager.readMetadata(); + TranslogTransferMetadata translogMetadata = translogTransferManager.readMetadata(timestamp); if (translogMetadata != null) { if (Files.notExists(location)) { Files.createDirectories(location); @@ -535,8 +617,31 @@ protected Releasable drainSync() { @Override public void trimUnreferencedReaders() throws IOException { - // clean up local translog files and updates readers - super.trimUnreferencedReaders(); + trimUnreferencedReaders(false, true); + } + + // Visible for testing + protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException { + if (trimLocal) { + // clean up local translog files and updates readers + super.trimUnreferencedReaders(); + } + + // Update file tracker to reflect local translog state + Optional minLiveGeneration = readers.stream().map(BaseTranslogReader::getGeneration).min(Long::compareTo); + if (minLiveGeneration.isPresent()) { + List staleFilesInTracker = new ArrayList<>(); + for (String file : fileTransferTracker.allUploaded()) { + if (file.endsWith(TRANSLOG_FILE_SUFFIX)) { + long generation = Translog.parseIdFromFileName(file); + if (generation < minLiveGeneration.get()) { + staleFilesInTracker.add(file); + staleFilesInTracker.add(Translog.getCommitCheckpointFileName(generation)); + } + } + fileTransferTracker.delete(staleFilesInTracker); + } + } // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. @@ -544,6 +649,12 @@ public void trimUnreferencedReaders() throws IOException { return; } + // This is to fail fast and avoid listing md files un-necessarily. + if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { + logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + return; + } + // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time. // Remote generations involves 2 async operations - 1) Delete translog generation files 2) Delete metadata files // We try to acquire 2 permits and if we can not, we return from here itself. @@ -551,34 +662,209 @@ public void trimUnreferencedReaders() throws IOException { return; } - // cleans up remote translog files not referenced in latest uploaded metadata. - // This enables us to restore translog from the metadata in case of failover or relocation. - Set generationsToDelete = new HashSet<>(); - for (long generation = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); generation >= 0; generation--) { - if (fileTransferTracker.uploaded(Translog.getFilename(generation)) == false) { - break; + ActionListener> listMetadataFilesListener = new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + + try { + if (metadataFiles.size() <= 1) { + logger.debug("No stale translog metadata files found"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + // Check last fetch status of pinned timestamps. If stale, return. + if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { + logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles); + + // If index is not deleted, make sure to keep latest metadata file + if (indexDeleted == false) { + metadataFilesToBeDeleted.remove(metadataFiles.get(0)); + } + + if (metadataFilesToBeDeleted.isEmpty()) { + logger.debug("No metadata files to delete"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + logger.debug("metadataFilesToBeDeleted = {}", metadataFilesToBeDeleted); + // For all the files that we are keeping, fetch min and max generations + List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + + logger.debug("metadataFilesNotToBeDeleted = {}", metadataFilesNotToBeDeleted); + Set generationsToBeDeleted = getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + indexDeleted + ); + + logger.debug("generationsToBeDeleted = {}", generationsToBeDeleted); + if (generationsToBeDeleted.isEmpty() == false) { + // Delete stale generations + translogTransferManager.deleteGenerationAsync( + primaryTermSupplier.getAsLong(), + generationsToBeDeleted, + remoteGenerationDeletionPermits::release + ); + + // Delete stale metadata files + translogTransferManager.deleteMetadataFilesAsync( + metadataFilesToBeDeleted, + remoteGenerationDeletionPermits::release + ); + + // Update cache to keep only those metadata files that are not getting deleted + oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); + + // Delete stale primary terms + deleteStaleRemotePrimaryTerms(metadataFiles); + } else { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + } + } catch (Exception e) { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + } } - generationsToDelete.add(generation); + + @Override + public void onFailure(Exception e) { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + logger.error("Exception while listing translog metadata files", e); + } + }; + translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + } + + // Visible for testing + protected Set getGenerationsToBeDeleted( + List metadataFilesNotToBeDeleted, + List metadataFilesToBeDeleted, + boolean indexDeleted + ) throws IOException { + long maxGenerationToBeDeleted = Long.MAX_VALUE; + + if (indexDeleted == false) { + maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); } - if (generationsToDelete.isEmpty() == false) { - deleteRemoteGeneration(generationsToDelete); - translogTransferManager.deleteStaleTranslogMetadataFilesAsync(remoteGenerationDeletionPermits::release); - deleteStaleRemotePrimaryTerms(); - } else { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + + Set generationsFromMetadataFilesToBeDeleted = new HashSet<>(); + for (String mdFile : metadataFilesToBeDeleted) { + Tuple minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager); + generationsFromMetadataFilesToBeDeleted.addAll( + LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList()) + ); + } + + Map> metadataFileNotToBeDeletedGenerationMap = getGenerationForMetadataFiles(metadataFilesNotToBeDeleted); + TreeSet> pinnedGenerations = getOrderedPinnedMetadataGenerations(metadataFileNotToBeDeletedGenerationMap); + Set generationsToBeDeleted = new HashSet<>(); + for (long generation : generationsFromMetadataFilesToBeDeleted) { + // Check if the generation is not referred by metadata file matching pinned timestamps + if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) { + generationsToBeDeleted.add(generation); + } } + return generationsToBeDeleted; } - /** - * Deletes remote translog and metadata files asynchronously corresponding to the generations. - * @param generations generations to be deleted. - */ - private void deleteRemoteGeneration(Set generations) { - translogTransferManager.deleteGenerationAsync( - primaryTermSupplier.getAsLong(), - generations, - remoteGenerationDeletionPermits::release + // Visible for testing + protected List getMetadataFilesToBeDeleted(List metadataFiles) { + Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); + + // Keep files since last successful run of scheduler + List metadataFilesToBeDeleted = RemoteStoreUtils.filterOutMetadataFilesBasedOnAge( + metadataFiles, + file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), + pinnedTimestampsState.v1() + ); + + logger.trace( + "metadataFiles.size = {}, metadataFilesToBeDeleted based on age based filtering = {}", + metadataFiles.size(), + metadataFilesToBeDeleted.size() ); + + // Get md files matching pinned timestamps + Set implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles( + metadataFilesToBeDeleted, + pinnedTimestampsState.v2(), + metadataFilePinnedTimestampMap, + file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ); + + // Filter out metadata files matching pinned timestamps + metadataFilesToBeDeleted.removeAll(implicitLockedFiles); + + logger.trace( + "implicitLockedFiles.size = {}, metadataFilesToBeDeleted based on pinned timestamp filtering = {}", + implicitLockedFiles.size(), + metadataFilesToBeDeleted.size() + ); + + return metadataFilesToBeDeleted; + } + + // Visible for testing + protected boolean isGenerationPinned(long generation, TreeSet> pinnedGenerations) { + Tuple ceilingGenerationRange = pinnedGenerations.ceiling(new Tuple<>(generation, generation)); + if (ceilingGenerationRange != null && generation >= ceilingGenerationRange.v1() && generation <= ceilingGenerationRange.v2()) { + return true; + } + Tuple floorGenerationRange = pinnedGenerations.floor(new Tuple<>(generation, generation)); + if (floorGenerationRange != null && generation >= floorGenerationRange.v1() && generation <= floorGenerationRange.v2()) { + return true; + } + return false; + } + + private TreeSet> getOrderedPinnedMetadataGenerations(Map> metadataFileGenerationMap) { + TreeSet> pinnedGenerations = new TreeSet<>((o1, o2) -> { + if (Objects.equals(o1.v1(), o2.v1()) == false) { + return o1.v1().compareTo(o2.v1()); + } else { + return o1.v2().compareTo(o2.v2()); + } + }); + pinnedGenerations.addAll(metadataFileGenerationMap.values()); + return pinnedGenerations; + } + + // Visible for testing + protected Map> getGenerationForMetadataFiles(List metadataFiles) throws IOException { + Map> metadataFileGenerationMap = new HashMap<>(); + for (String metadataFile : metadataFiles) { + metadataFileGenerationMap.put(metadataFile, getMinMaxTranslogGenerationFromMetadataFile(metadataFile, translogTransferManager)); + } + return metadataFileGenerationMap; + } + + // Visible for testing + protected Tuple getMinMaxTranslogGenerationFromMetadataFile( + String metadataFile, + TranslogTransferManager translogTransferManager + ) throws IOException { + Tuple minMaxGenerationFromFileName = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(metadataFile); + if (minMaxGenerationFromFileName != null) { + return minMaxGenerationFromFileName; + } else { + if (oldFormatMetadataFileGenerationMap.containsKey(metadataFile)) { + return oldFormatMetadataFileGenerationMap.get(metadataFile); + } else { + TranslogTransferMetadata metadata = translogTransferManager.readMetadata(metadataFile); + Tuple minMaxGenTuple = new Tuple<>(metadata.getMinTranslogGeneration(), metadata.getGeneration()); + oldFormatMetadataFileGenerationMap.put(metadataFile, minMaxGenTuple); + return minMaxGenTuple; + } + } } /** @@ -587,17 +873,20 @@ private void deleteRemoteGeneration(Set generations) { *
* This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ - private void deleteStaleRemotePrimaryTerms() { + private void deleteStaleRemotePrimaryTerms(List metadataFiles) { // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. if (olderPrimaryCleaned.trySet(Boolean.TRUE)) { - if (readers.isEmpty()) { - logger.trace("Translog reader list is empty, returning from deleteStaleRemotePrimaryTerms"); + if (metadataFiles.isEmpty()) { + logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); return; } + Optional minPrimaryTerm = metadataFiles.stream() + .map(file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[1])) + .min(Long::compareTo); // First we delete all stale primary terms folders from remote store - long minimumReferencedPrimaryTerm = readers.stream().map(BaseTranslogReader::getPrimaryTerm).min(Long::compare).get(); + long minimumReferencedPrimaryTerm = minPrimaryTerm.get() - 1; translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); } } @@ -633,7 +922,15 @@ public static void cleanup( protected void onDelete() { ClusterService.assertClusterOrClusterManagerStateThread(); // clean up all remote translog files - translogTransferManager.delete(); + if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { + try { + trimUnreferencedReaders(true, false); + } catch (IOException e) { + logger.error("Exception while deleting translog files from remote store", e); + } + } else { + translogTransferManager.delete(); + } } // Visible for testing diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 87e0c21b8203c..4b4ceb7444471 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -317,6 +317,10 @@ TranslogReader openReader(Path path, Checkpoint checkpoint) throws IOException { */ public static long parseIdFromFileName(Path translogFile) { final String fileName = translogFile.getFileName().toString(); + return parseIdFromFileName(fileName); + } + + public static long parseIdFromFileName(String fileName) { final Matcher matcher = PARSE_STRICT_ID_PATTERN.matcher(fileName); if (matcher.matches()) { try { diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 1cc39cdf442e2..527fb0123d800 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -45,10 +45,12 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import static org.opensearch.index.translog.transfer.FileSnapshot.TranslogFileSnapshot; +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; /** * The class responsible for orchestrating the transfer of a {@link TransferSnapshot} via a {@link TransferService} @@ -337,35 +339,54 @@ private void deleteFileIfExists(Path filePath) throws IOException { } } + public TranslogTransferMetadata readMetadata(long pinnedTimestamp) throws IOException { + if (pinnedTimestamp <= 0) { + return readMetadata(); + } + return readMetadata((blobMetadataList) -> { + List metadataFiles = blobMetadataList.stream().map(BlobMetadata::name).collect(Collectors.toList()); + Set metadataFilesMatchingTimestamp = RemoteStoreUtils.getPinnedTimestampLockedFiles( + metadataFiles, + Set.of(pinnedTimestamp), + file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen, + true + ); + if (metadataFilesMatchingTimestamp.isEmpty()) { + return null; + } + assert metadataFilesMatchingTimestamp.size() == 1 : "There should be only 1 metadata file matching given timestamp"; + return metadataFilesMatchingTimestamp.stream().findFirst().get(); + }, Integer.MAX_VALUE); + } + public TranslogTransferMetadata readMetadata() throws IOException { + return readMetadata((blobMetadataList) -> { + RemoteStoreUtils.verifyNoMultipleWriters( + blobMetadataList.stream().map(BlobMetadata::name).collect(Collectors.toList()), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ); + return blobMetadataList.get(0).name(); + }, METADATA_FILES_TO_FETCH); + } + + private TranslogTransferMetadata readMetadata(Function, String> getMetadataFileToRead, int numberOfFilesToFetch) + throws IOException { SetOnce metadataSetOnce = new SetOnce<>(); SetOnce exceptionSetOnce = new SetOnce<>(); final CountDownLatch latch = new CountDownLatch(1); LatchedActionListener> latchedActionListener = new LatchedActionListener<>( ActionListener.wrap(blobMetadataList -> { if (blobMetadataList.isEmpty()) return; - RemoteStoreUtils.verifyNoMultipleWriters( - blobMetadataList.stream().map(BlobMetadata::name).collect(Collectors.toList()), - TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen - ); - String filename = blobMetadataList.get(0).name(); - boolean downloadStatus = false; - long downloadStartTime = System.nanoTime(), bytesToRead = 0; - try (InputStream inputStream = transferService.downloadBlob(remoteMetadataTransferPath, filename)) { - // Capture number of bytes for stats before reading - bytesToRead = inputStream.available(); - IndexInput indexInput = new ByteArrayIndexInput("metadata file", inputStream.readAllBytes()); - metadataSetOnce.set(metadataStreamWrapper.readStream(indexInput)); - downloadStatus = true; + String filename = getMetadataFileToRead.apply(blobMetadataList); + if (filename == null) { + return; + } + try { + metadataSetOnce.set(readMetadata(filename)); } catch (IOException e) { logger.error(() -> new ParameterizedMessage("Exception while reading metadata file: {}", filename), e); exceptionSetOnce.set(e); - } finally { - remoteTranslogTransferTracker.addDownloadTimeInMillis((System.nanoTime() - downloadStartTime) / 1_000_000L); - logger.debug("translogMetadataDownloadStatus={}", downloadStatus); - if (downloadStatus) { - remoteTranslogTransferTracker.addDownloadBytesSucceeded(bytesToRead); - } } }, e -> { if (e instanceof RuntimeException) { @@ -381,12 +402,14 @@ public TranslogTransferMetadata readMetadata() throws IOException { transferService.listAllInSortedOrder( remoteMetadataTransferPath, TranslogTransferMetadata.METADATA_PREFIX, - METADATA_FILES_TO_FETCH, + numberOfFilesToFetch, latchedActionListener ); - latch.await(); + if (latch.await(remoteStoreSettings.getClusterRemoteTranslogTransferTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + throw new RuntimeException("Timed out reading metadata file"); + } } catch (InterruptedException e) { - throw new IOException("Exception while reading/downloading metadafile", e); + throw new IOException("Exception while reading/downloading metadata file", e); } if (exceptionSetOnce.get() != null) { @@ -396,6 +419,26 @@ public TranslogTransferMetadata readMetadata() throws IOException { return metadataSetOnce.get(); } + public TranslogTransferMetadata readMetadata(String metadataFilename) throws IOException { + boolean downloadStatus = false; + TranslogTransferMetadata translogTransferMetadata = null; + long downloadStartTime = System.nanoTime(), bytesToRead = 0; + try (InputStream inputStream = transferService.downloadBlob(remoteMetadataTransferPath, metadataFilename)) { + // Capture number of bytes for stats before reading + bytesToRead = inputStream.available(); + IndexInput indexInput = new ByteArrayIndexInput("metadata file", inputStream.readAllBytes()); + translogTransferMetadata = metadataStreamWrapper.readStream(indexInput); + downloadStatus = true; + } finally { + remoteTranslogTransferTracker.addDownloadTimeInMillis((System.nanoTime() - downloadStartTime) / 1_000_000L); + logger.debug("translogMetadataDownloadStatus={}", downloadStatus); + if (downloadStatus) { + remoteTranslogTransferTracker.addDownloadBytesSucceeded(bytesToRead); + } + } + return translogTransferMetadata; + } + private TransferFileSnapshot prepareMetadata(TransferSnapshot transferSnapshot) throws IOException { Map generationPrimaryTermMap = transferSnapshot.getTranslogFileSnapshots().stream().map(s -> { assert s instanceof TranslogFileSnapshot; @@ -549,6 +592,16 @@ public void onFailure(Exception e) { }); } + public void listTranslogMetadataFilesAsync(ActionListener> listener) { + transferService.listAllInSortedOrderAsync( + ThreadPool.Names.REMOTE_PURGE, + remoteMetadataTransferPath, + TranslogTransferMetadata.METADATA_PREFIX, + Integer.MAX_VALUE, + listener + ); + } + public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) { try { transferService.listAllInSortedOrderAsync( @@ -635,7 +688,7 @@ public void onFailure(Exception e) { * @param files list of metadata files to be deleted. * @param onCompletion runnable to run on completion of deletion regardless of success/failure. */ - private void deleteMetadataFilesAsync(List files, Runnable onCompletion) { + public void deleteMetadataFilesAsync(List files, Runnable onCompletion) { try { transferService.deleteBlobsAsync(ThreadPool.Names.REMOTE_PURGE, remoteMetadataTransferPath, files, new ActionListener<>() { @Override diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index acd7574e648a3..745fa9a8a219a 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -8,6 +8,10 @@ package org.opensearch.index.translog.transfer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.Version; import org.opensearch.common.SetOnce; import org.opensearch.common.collect.Tuple; import org.opensearch.index.remote.RemoteStoreUtils; @@ -25,6 +29,8 @@ */ public class TranslogTransferMetadata { + public static final Logger logger = LogManager.getLogger(TranslogTransferMetadata.class); + private final long primaryTerm; private final long generation; @@ -128,6 +134,24 @@ public static Tuple getNodeIdByPrimaryTermAndGen(String filename return new Tuple<>(primaryTermAndGen, nodeId); } + public static Tuple getMinMaxTranslogGenerationFromFilename(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + if (tokens.length < 7) { + // For versions < 2.17, we don't have min translog generation. + return null; + } + assert Version.CURRENT.onOrAfter(Version.V_2_17_0); + try { + // instead of direct index, we go backwards to avoid running into same separator in nodeId + String minGeneration = tokens[tokens.length - 2]; + String maxGeneration = tokens[2]; + return new Tuple<>(RemoteStoreUtils.invertLong(minGeneration), RemoteStoreUtils.invertLong(maxGeneration)); + } catch (NumberFormatException e) { + logger.error(() -> new ParameterizedMessage("Exception while getting min and max translog generation from: {}", filename), e); + return null; + } + } + @Override public int hashCode() { return Objects.hash(primaryTerm, generation); diff --git a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java index 55280ca5c96d6..00d9ffdac6086 100644 --- a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java +++ b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java @@ -292,6 +292,11 @@ public static TimeValue getPinnedTimestampsLookbackInterval() { return pinnedTimestampsLookbackInterval; } + // Visible for testing + public static void setPinnedTimestampsLookbackInterval(TimeValue pinnedTimestampsLookbackInterval) { + RemoteStoreSettings.pinnedTimestampsLookbackInterval = pinnedTimestampsLookbackInterval; + } + public static boolean isPinnedTimestampsEnabled() { return isPinnedTimestampsEnabled; } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 6bf35cc1eac9b..163340e8ec7d5 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -121,12 +121,12 @@ public class RemoteFsTranslogTests extends OpenSearchTestCase { private AtomicLong globalCheckpoint; protected Path translogDir; // A default primary term is used by translog instances created in this test. - private final AtomicLong primaryTerm = new AtomicLong(); + protected final AtomicLong primaryTerm = new AtomicLong(); private final AtomicBoolean primaryMode = new AtomicBoolean(true); private final AtomicReference persistedSeqNoConsumer = new AtomicReference<>(); - private ThreadPool threadPool; - private final static String METADATA_DIR = "metadata"; - private final static String DATA_DIR = "data"; + protected ThreadPool threadPool; + protected final static String METADATA_DIR = "metadata"; + protected final static String DATA_DIR = "data"; AtomicInteger writeCalls = new AtomicInteger(); BlobStoreRepository repository; @@ -277,7 +277,7 @@ private Translog.Location addToTranslogAndList(Translog translog, List list, Translog.Operation op) + protected Translog.Location addToTranslogAndListAndUpload(Translog translog, List list, Translog.Operation op) throws IOException { Translog.Location loc = translog.add(op); translog.ensureSynced(loc); @@ -390,7 +390,7 @@ public void testSimpleOperations() throws IOException { } - private TranslogConfig getConfig(int gensToKeep) { + protected TranslogConfig getConfig(int gensToKeep) { Path tempDir = createTempDir(); final TranslogConfig temp = getTranslogConfig(tempDir, gensToKeep); final TranslogConfig config = new TranslogConfig( @@ -405,7 +405,7 @@ private TranslogConfig getConfig(int gensToKeep) { return config; } - private ChannelFactory getChannelFactory() { + protected ChannelFactory getChannelFactory() { writeCalls = new AtomicInteger(); final ChannelFactory channelFactory = (file, openOption) -> { FileChannel delegate = FileChannel.open(file, openOption); @@ -493,8 +493,8 @@ ChannelFactory getChannelFactory() { 6, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); - } + } public void testReadLocation() throws IOException { @@ -671,13 +671,13 @@ public void testSimpleOperationsUpload() throws Exception { assertThat(snapshot.totalOperations(), equalTo(ops.size())); } - assertEquals(4, translog.allUploaded().size()); + assertEquals(2, translog.allUploaded().size()); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); - assertEquals(6, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); translog.rollGeneration(); - assertEquals(6, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); Set mdFiles = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); assertEquals(2, mdFiles.size()); @@ -736,7 +736,7 @@ public void testSimpleOperationsUpload() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); assertBusy(() -> { - assertEquals(4, translog.allUploaded().size()); + assertEquals(2, translog.allUploaded().size()); assertEquals( 4, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() @@ -755,13 +755,12 @@ public void testSimpleOperationsUpload() throws Exception { assertEquals(1, translog.readers.size()); assertEquals(1, translog.stats().estimatedNumberOfOperations()); assertBusy(() -> { - assertEquals(4, translog.allUploaded().size()); + assertEquals(2, translog.allUploaded().size()); assertEquals( 4, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); }); - } public void testMetadataFileDeletion() throws Exception { @@ -775,7 +774,7 @@ public void testMetadataFileDeletion() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); } - assertBusy(() -> assertEquals(4, translog.allUploaded().size())); + assertBusy(() -> assertEquals(2, translog.allUploaded().size())); assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); int moreDocs = randomIntBetween(3, 10); logger.info("numDocs={} moreDocs={}", numDocs, moreDocs); @@ -873,7 +872,7 @@ public void testDrainSync() throws Exception { assertBusy(() -> assertEquals(0, latch.getCount())); assertEquals(0, translog.availablePermits()); slowDown.setSleepSeconds(0); - assertEquals(6, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); assertEquals(2, translog.readers.size()); Set mdFiles = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); @@ -882,7 +881,7 @@ public void testDrainSync() throws Exception { translog.trimUnreferencedReaders(); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); - assertEquals(6, translog.allUploaded().size()); + assertEquals(2, translog.allUploaded().size()); assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); // Case 4 - After drainSync, if an upload is an attempted, we do not upload to remote store. @@ -892,25 +891,25 @@ public void testDrainSync() throws Exception { new Translog.Index(String.valueOf(2), 2, primaryTerm.get(), new byte[] { 1 }) ); assertEquals(1, translog.readers.size()); - assertEquals(6, translog.allUploaded().size()); + assertEquals(2, translog.allUploaded().size()); assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); // Refill the permits back Releasables.close(releasable); addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(3), 3, primaryTerm.get(), new byte[] { 1 })); assertEquals(2, translog.readers.size()); - assertEquals(8, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); translog.setMinSeqNoToKeep(3); translog.trimUnreferencedReaders(); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); - assertBusy(() -> assertEquals(4, translog.allUploaded().size())); + assertBusy(() -> assertEquals(2, translog.allUploaded().size())); assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); } - private BlobPath getTranslogDirectory() { + protected BlobPath getTranslogDirectory() { return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG.getName()); } @@ -1688,13 +1687,13 @@ public void testDownloadWithRetries() throws IOException { TranslogTransferManager mockTransfer = mock(TranslogTransferManager.class); RemoteTranslogTransferTracker remoteTranslogTransferTracker = mock(RemoteTranslogTransferTracker.class); - when(mockTransfer.readMetadata()).thenReturn(translogTransferMetadata); + when(mockTransfer.readMetadata(0)).thenReturn(translogTransferMetadata); when(mockTransfer.getRemoteTranslogTransferTracker()).thenReturn(remoteTranslogTransferTracker); // Always File not found when(mockTransfer.downloadTranslog(any(), any(), any())).thenThrow(new NoSuchFileException("File not found")); TranslogTransferManager finalMockTransfer = mockTransfer; - assertThrows(NoSuchFileException.class, () -> RemoteFsTranslog.download(finalMockTransfer, location, logger, false)); + assertThrows(NoSuchFileException.class, () -> RemoteFsTranslog.download(finalMockTransfer, location, logger, false, 0)); // File not found in first attempt . File found in second attempt. mockTransfer = mock(TranslogTransferManager.class); @@ -1715,7 +1714,7 @@ public void testDownloadWithRetries() throws IOException { }).when(mockTransfer).downloadTranslog(any(), any(), any()); // no exception thrown - RemoteFsTranslog.download(mockTransfer, location, logger, false); + RemoteFsTranslog.download(mockTransfer, location, logger, false, 0); } // No translog data in local as well as remote, we skip creating empty translog @@ -1728,7 +1727,7 @@ public void testDownloadWithNoTranslogInLocalAndRemote() throws IOException { when(mockTransfer.getRemoteTranslogTransferTracker()).thenReturn(remoteTranslogTransferTracker); Path[] filesBeforeDownload = FileSystemUtils.files(location); - RemoteFsTranslog.download(mockTransfer, location, logger, false); + RemoteFsTranslog.download(mockTransfer, location, logger, false, 0); assertEquals(filesBeforeDownload, FileSystemUtils.files(location)); } @@ -1748,7 +1747,7 @@ public void testDownloadWithTranslogOnlyInLocal() throws IOException { Checkpoint existingCheckpoint = Translog.readCheckpoint(location); TranslogTransferManager finalMockTransfer = mockTransfer; - RemoteFsTranslog.download(finalMockTransfer, location, logger, false); + RemoteFsTranslog.download(finalMockTransfer, location, logger, false, 0); Path[] filesPostDownload = FileSystemUtils.files(location); assertEquals(2, filesPostDownload.length); @@ -1784,11 +1783,11 @@ public void testDownloadWithEmptyTranslogOnlyInLocal() throws IOException { TranslogTransferManager finalMockTransfer = mockTransfer; // download first time will ensure creating empty translog - RemoteFsTranslog.download(finalMockTransfer, location, logger, false); + RemoteFsTranslog.download(finalMockTransfer, location, logger, false, 0); Path[] filesPostFirstDownload = FileSystemUtils.files(location); // download on empty translog should be a no-op - RemoteFsTranslog.download(finalMockTransfer, location, logger, false); + RemoteFsTranslog.download(finalMockTransfer, location, logger, false, 0); Path[] filesPostSecondDownload = FileSystemUtils.files(location); assertArrayEquals(filesPostFirstDownload, filesPostSecondDownload); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java new file mode 100644 index 0000000000000..e35ab6bcd14c3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java @@ -0,0 +1,840 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.lease.Releasables; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.model.RemotePinnedTimestamps; +import org.opensearch.gateway.remote.model.RemoteStorePinnedTimestampsBlobStore; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.remote.RemoteTranslogTransferTracker; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.index.translog.transfer.TranslogTransferManager; +import org.opensearch.index.translog.transfer.TranslogTransferMetadata; +import org.opensearch.indices.DefaultRemoteStoreSettings; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.BlobStoreTestUtil; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.threadpool.ThreadPool; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import org.mockito.Mockito; + +import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; +import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@LuceneTestCase.SuppressFileSystems("ExtrasFS") +public class RemoteFsTranslogWithPinnedTimestampTests extends RemoteFsTranslogTests { + + Runnable updatePinnedTimstampTask; + BlobStoreTransferService pinnedTimestampBlobStoreTransferService; + RemoteStorePinnedTimestampsBlobStore remoteStorePinnedTimestampsBlobStore; + RemoteStorePinnedTimestampService remoteStorePinnedTimestampServiceSpy; + + @Before + public void setUp() throws Exception { + super.setUp(); + + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings( + Settings.builder().put(CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true).build(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + + Supplier repositoriesServiceSupplier = mock(Supplier.class); + Settings settings = Settings.builder() + .put(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote-repo") + .build(); + RepositoriesService repositoriesService = mock(RepositoriesService.class); + when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); + BlobStoreRepository blobStoreRepository = mock(BlobStoreRepository.class); + when(repositoriesService.repository("remote-repo")).thenReturn(blobStoreRepository); + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.schedule(any(), any(), any())).then(invocationOnMock -> { + Runnable updateTask = invocationOnMock.getArgument(0); + updatePinnedTimstampTask = () -> { + long currentTime = System.currentTimeMillis(); + while (RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() < currentTime) { + updateTask.run(); + } + }; + return null; + }).then(subsequentInvocationsOnMock -> null); + + RepositoryMetadata repositoryMetadata = new RepositoryMetadata(randomAlphaOfLength(10), FsRepository.TYPE, settings); + final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(repositoryMetadata); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = new RemoteStorePinnedTimestampService( + repositoriesServiceSupplier, + settings, + threadPool, + clusterService + ); + remoteStorePinnedTimestampServiceSpy = Mockito.spy(remoteStorePinnedTimestampService); + + remoteStorePinnedTimestampsBlobStore = mock(RemoteStorePinnedTimestampsBlobStore.class); + pinnedTimestampBlobStoreTransferService = mock(BlobStoreTransferService.class); + when(remoteStorePinnedTimestampServiceSpy.pinnedTimestampsBlobStore()).thenReturn(remoteStorePinnedTimestampsBlobStore); + when(remoteStorePinnedTimestampServiceSpy.blobStoreTransferService()).thenReturn(pinnedTimestampBlobStoreTransferService); + + doAnswer(invocationOnMock -> { + ActionListener> actionListener = invocationOnMock.getArgument(3); + actionListener.onResponse(new ArrayList<>()); + return null; + }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + + remoteStorePinnedTimestampServiceSpy.start(); + } + + public void testGetMinMaxTranslogGenerationFromFilename() throws Exception { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + + CountDownLatch latch = new CountDownLatch(1); + blobStoreTransferService.listAllInSortedOrder( + getTranslogDirectory().add(METADATA_DIR), + "metadata", + Integer.MAX_VALUE, + new LatchedActionListener<>(new ActionListener>() { + @Override + public void onResponse(List blobMetadataList) { + Long minGen = 1L; + Long maxGen = 6L; + for (BlobMetadata blobMetadata : blobMetadataList) { + Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(blobMetadata.name()); + assertEquals(minGen, minMaxGen.v1()); + assertEquals(maxGen, minMaxGen.v2()); + maxGen -= 1; + } + } + + @Override + public void onFailure(Exception e) { + // This means test failure + fail(); + } + }, latch) + ); + latch.await(); + + // Old format metadata file + String oldFormatMdFilename = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"; + assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(oldFormatMdFilename)); + + // Node id containing separator + String nodeIdWithSeparator = + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node__1__9223372036438563958__1"; + Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(nodeIdWithSeparator); + Long minGen = Long.MAX_VALUE - 9223372036438563958L; + assertEquals(minGen, minMaxGen.v1()); + + // Malformed md filename + String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__1"; + assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(malformedMdFileName)); + } + + public void testIndexDeletionWithNoPinnedTimestampNoRecentMdFiles() throws Exception { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> { + assertEquals(5, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 12, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(true, false); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + assertBusy(() -> { + assertEquals(0, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 0, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + } + + public void testIndexDeletionWithNoPinnedTimestampButRecentFiles() throws Exception { + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(true, false); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertBusy(() -> { + assertEquals(5, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 12, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + } + + @Override + public void testSimpleOperationsUpload() throws Exception { + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + + // First reader is created at the init of translog + assertEquals(3, translog.readers.size()); + assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertBusy(() -> { + assertEquals(6, translog.allUploaded().size()); + assertEquals( + 6, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> { + assertEquals( + 16, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + + translog.setMinSeqNoToKeep(4); + translog.trimUnreferencedReaders(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 })); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals(5, translog.readers.size()); + assertBusy(() -> { + assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals(10, translog.allUploaded().size()); + assertEquals( + 10, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + } + + @Override + public void testMetadataFileDeletion() throws Exception { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + ArrayList ops = new ArrayList<>(); + // Test deletion of metadata files + int numDocs = randomIntBetween(6, 10); + for (int i = 0; i < numDocs; i++) { + addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(i), i, primaryTerm.get(), new byte[] { 1 })); + translog.setMinSeqNoToKeep(i); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals(1, translog.readers.size()); + } + assertBusy(() -> assertEquals(2, translog.allUploaded().size())); + addToTranslogAndListAndUpload( + translog, + ops, + new Translog.Index(String.valueOf(numDocs), numDocs, primaryTerm.get(), new byte[] { 1 }) + ); + addToTranslogAndListAndUpload( + translog, + ops, + new Translog.Index(String.valueOf(numDocs + 1), numDocs + 1, primaryTerm.get(), new byte[] { 1 }) + ); + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> { assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); }); + } + + public void testMetadataFileDeletionWithPinnedTimestamps() throws Exception { + ArrayList ops = new ArrayList<>(); + // Test deletion of metadata files + int numDocs = randomIntBetween(16, 20); + for (int i = 0; i < numDocs; i++) { + addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(i), i, primaryTerm.get(), new byte[] { 1 })); + translog.setMinSeqNoToKeep(i); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + translog.trimUnreferencedReaders(); + // This is just to make sure that each metadata is at least 1ms apart + Thread.sleep(1); + } + + CountDownLatch latch = new CountDownLatch(1); + blobStoreTransferService.listAllInSortedOrder( + getTranslogDirectory().add(METADATA_DIR), + "metadata", + Integer.MAX_VALUE, + new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(List blobMetadataList) { + List pinnedTimestampMatchingMetadataFiles = new ArrayList<>(); + List pinnedTimestamps = new ArrayList<>(); + for (BlobMetadata blobMetadata : blobMetadataList) { + String metadataFilename = blobMetadata.name(); + if (randomBoolean()) { + long timestamp = RemoteStoreUtils.invertLong(metadataFilename.split(METADATA_SEPARATOR)[3]); + pinnedTimestamps.add(timestamp); + pinnedTimestampMatchingMetadataFiles.add(metadataFilename); + } + } + + doAnswer(invocationOnMock -> { + ActionListener> actionListener = invocationOnMock.getArgument(3); + actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); + return null; + }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + + Map> pinnedTimestampsMap = new HashMap<>(); + pinnedTimestamps.forEach(ts -> pinnedTimestampsMap.put(ts, new ArrayList<>())); + + try { + when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( + new RemotePinnedTimestamps.PinnedTimestamps(pinnedTimestampsMap) + ); + when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + + Set dataFilesBeforeTrim = blobStoreTransferService.listAll( + getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get())) + ); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + updatePinnedTimstampTask.run(); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + Set metadataFilesAfterTrim = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); + Set dataFilesAfterTrim = blobStoreTransferService.listAll( + getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get())) + ); + + // If non pinned generations are within, minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep() + // we will not delete them + if (dataFilesAfterTrim.equals(dataFilesBeforeTrim) == false) { + // We check for number of pinned timestamp or +1 due to latest metadata. + assertTrue( + metadataFilesAfterTrim.size() == pinnedTimestamps.size() + || metadataFilesAfterTrim.size() == pinnedTimestamps.size() + 1 + ); + } + + for (String md : pinnedTimestampMatchingMetadataFiles) { + assertTrue(metadataFilesAfterTrim.contains(md)); + Tuple minMaXGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(md); + for (long i = minMaXGen.v1(); i <= minMaXGen.v2(); i++) { + assertTrue(dataFilesAfterTrim.contains(Translog.getFilename(i))); + } + } + } catch (Exception e) { + fail(); + } + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }, latch) + ); + + latch.await(); + } + + @Override + public void testDrainSync() throws Exception { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + // This test checks following scenarios - + // 1. During ongoing uploads, the available permits are 0. + // 2. During an upload, if drainSync is called, it will wait for it to acquire and available permits are 0. + // 3. After drainSync, if trimUnreferencedReaders is attempted, we do not delete from remote store. + // 4. After drainSync, if an upload is an attempted, we do not upload to remote store. + ArrayList ops = new ArrayList<>(); + assertEquals(0, translog.allUploaded().size()); + assertEquals(1, translog.readers.size()); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(0), 0, primaryTerm.get(), new byte[] { 1 })); + assertEquals(4, translog.allUploaded().size()); + assertEquals(2, translog.readers.size()); + assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + + translog.setMinSeqNoToKeep(0); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals(1, translog.readers.size()); + + // Case 1 - During ongoing uploads, the available permits are 0. + slowDown.setSleepSeconds(2); + CountDownLatch latch = new CountDownLatch(1); + Thread thread1 = new Thread(() -> { + try { + addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(1), 1, primaryTerm.get(), new byte[] { 1 })); + assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + latch.countDown(); + } catch (IOException e) { + throw new AssertionError(e); + } + }); + thread1.start(); + assertBusy(() -> assertEquals(0, translog.availablePermits())); + // Case 2 - During an upload, if drainSync is called, it will wait for it to acquire and available permits are 0. + Releasable releasable = translog.drainSync(); + assertBusy(() -> assertEquals(0, latch.getCount())); + assertEquals(0, translog.availablePermits()); + slowDown.setSleepSeconds(0); + assertEquals(4, translog.allUploaded().size()); + assertEquals(2, translog.readers.size()); + Set mdFiles = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); + + // Case 3 - After drainSync, if trimUnreferencedReaders is attempted, we do not delete from remote store. + translog.setMinSeqNoToKeep(1); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals(1, translog.readers.size()); + assertEquals(2, translog.allUploaded().size()); + assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); + + // Case 4 - After drainSync, if an upload is an attempted, we do not upload to remote store. + Translog.Location loc = addToTranslogAndListAndUpload( + translog, + ops, + new Translog.Index(String.valueOf(2), 2, primaryTerm.get(), new byte[] { 1 }) + ); + assertEquals(1, translog.readers.size()); + assertEquals(2, translog.allUploaded().size()); + assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); + + // Refill the permits back + Releasables.close(releasable); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(3), 3, primaryTerm.get(), new byte[] { 1 })); + assertEquals(2, translog.readers.size()); + assertEquals(4, translog.allUploaded().size()); + assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + + translog.setMinSeqNoToKeep(3); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals(1, translog.readers.size()); + assertBusy(() -> assertEquals(2, translog.allUploaded().size())); + assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + } + + @Override + public void testExtraGenToKeep() throws Exception { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + TranslogConfig config = getConfig(1); + ChannelFactory channelFactory = getChannelFactory(); + final Set persistedSeqNos = new HashSet<>(); + String translogUUID = Translog.createEmptyTranslog( + config.getTranslogPath(), + SequenceNumbers.NO_OPS_PERFORMED, + shardId, + channelFactory, + primaryTerm.get() + ); + TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(config.getIndexSettings()); + ArrayList ops = new ArrayList<>(); + try ( + RemoteFsTranslog translog = new RemoteFsTranslog( + config, + translogUUID, + deletionPolicy, + () -> SequenceNumbers.NO_OPS_PERFORMED, + primaryTerm::get, + persistedSeqNos::add, + repository, + threadPool, + () -> Boolean.TRUE, + new RemoteTranslogTransferTracker(shardId, 10), + DefaultRemoteStoreSettings.INSTANCE + ) { + @Override + ChannelFactory getChannelFactory() { + return channelFactory; + } + } + ) { + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 2, primaryTerm.get(), new byte[] { 1 })); + + // expose the new checkpoint (simulating a commit), before we trim the translog + translog.setMinSeqNoToKeep(2); + + // Trims from local + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 3, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 4, primaryTerm.get(), new byte[] { 1 })); + // Trims from remote now + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertEquals( + 8, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + } + } + + public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() throws IOException { + List metadataFilesNotToBeDeleted = new ArrayList<>(); + List metadataFilesToBeDeleted = List.of( + // 4 to 7 + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + // 17 to 37 + "metadata__9223372036438563903__9223372036854775770__9223370311919910398__31__9223372036854775790__1", + // 27 to 42 + "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" + ); + Set generations = translog.getGenerationsToBeDeleted(metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, true); + Set md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet()); + Set md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet()); + Set md3Generations = LongStream.rangeClosed(27, 42).boxed().collect(Collectors.toSet()); + assertTrue(generations.containsAll(md1Generations)); + assertTrue(generations.containsAll(md2Generations)); + assertTrue(generations.containsAll(md3Generations)); + + generations.removeAll(md1Generations); + generations.removeAll(md2Generations); + generations.removeAll(md3Generations); + assertTrue(generations.isEmpty()); + } + + public void testGetGenerationsToBeDeleted() throws IOException { + List metadataFilesNotToBeDeleted = List.of( + // 1 to 4 + "metadata__9223372036438563903__9223372036854775803__9223370311919910398__31__9223372036854775806__1", + // 26 to 30 + "metadata__9223372036438563903__9223372036854775777__9223370311919910398__31__9223372036854775781__1", + // 42 to 100 + "metadata__9223372036438563903__9223372036854775707__9223370311919910403__31__9223372036854775765__1" + ); + List metadataFilesToBeDeleted = List.of( + // 4 to 7 + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + // 17 to 37 + "metadata__9223372036438563903__9223372036854775770__9223370311919910398__31__9223372036854775790__1", + // 27 to 42 + "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" + ); + Set generations = translog.getGenerationsToBeDeleted(metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, true); + Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); + Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); + Set md3Generations = LongStream.rangeClosed(31, 41).boxed().collect(Collectors.toSet()); + assertTrue(generations.containsAll(md1Generations)); + assertTrue(generations.containsAll(md2Generations)); + assertTrue(generations.containsAll(md3Generations)); + + generations.removeAll(md1Generations); + generations.removeAll(md2Generations); + generations.removeAll(md3Generations); + assertTrue(generations.isEmpty()); + } + + public void testGetMetadataFilesToBeDeletedNoExclusion() { + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1" + ); + + assertEquals(metadataFiles, translog.getMetadataFilesToBeDeleted(metadataFiles)); + } + + public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { + updatePinnedTimstampTask.run(); + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 100000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000); + + List metadataFiles = List.of( + "metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775800__" + md2Timestamp + "__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + assertEquals(1, metadataFilesToBeDeleted.size()); + assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); + } + + public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 100000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + doAnswer(invocationOnMock -> { + ActionListener> actionListener = invocationOnMock.getArgument(3); + actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); + return null; + }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + + long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000; + when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( + new RemotePinnedTimestamps.PinnedTimestamps(Map.of(pinnedTimestamp, List.of("xyz"))) + ); + when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + "metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775600__" + md2Timestamp + "__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + assertEquals(2, metadataFilesToBeDeleted.size()); + assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); + } + + public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 100000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + doAnswer(invocationOnMock -> { + ActionListener> actionListener = invocationOnMock.getArgument(3); + actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); + return null; + }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + + long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000; + when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( + new RemotePinnedTimestamps.PinnedTimestamps(Map.of(pinnedTimestamp, List.of("xyz"))) + ); + when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + "metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775600__" + md2Timestamp + "__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + assertEquals(1, metadataFilesToBeDeleted.size()); + assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0)); + } + + public void testIsGenerationPinned() { + TreeSet> pinnedGenerations = new TreeSet<>(new TreeSet<>((o1, o2) -> { + if (Objects.equals(o1.v1(), o2.v1()) == false) { + return o1.v1().compareTo(o2.v1()); + } else { + return o1.v2().compareTo(o2.v2()); + } + })); + + pinnedGenerations.add(new Tuple<>(12L, 34L)); + pinnedGenerations.add(new Tuple<>(121L, 140L)); + pinnedGenerations.add(new Tuple<>(142L, 160L)); + pinnedGenerations.add(new Tuple<>(12L, 120L)); + pinnedGenerations.add(new Tuple<>(12L, 78L)); + pinnedGenerations.add(new Tuple<>(142L, 170L)); + pinnedGenerations.add(new Tuple<>(1L, 1L)); + pinnedGenerations.add(new Tuple<>(12L, 56L)); + pinnedGenerations.add(new Tuple<>(142L, 180L)); + pinnedGenerations.add(new Tuple<>(4L, 9L)); + + assertFalse(translog.isGenerationPinned(3, pinnedGenerations)); + assertFalse(translog.isGenerationPinned(10, pinnedGenerations)); + assertFalse(translog.isGenerationPinned(141, pinnedGenerations)); + assertFalse(translog.isGenerationPinned(181, pinnedGenerations)); + assertFalse(translog.isGenerationPinned(5000, pinnedGenerations)); + assertFalse(translog.isGenerationPinned(0, pinnedGenerations)); + + assertTrue(translog.isGenerationPinned(1, pinnedGenerations)); + assertTrue(translog.isGenerationPinned(120, pinnedGenerations)); + assertTrue(translog.isGenerationPinned(121, pinnedGenerations)); + assertTrue(translog.isGenerationPinned(156, pinnedGenerations)); + assertTrue(translog.isGenerationPinned(12, pinnedGenerations)); + } + + public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + // Fetch generations directly from the filename + assertEquals( + new Tuple<>(701L, 1008L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(4L, 7L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(106L, 106L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(4573L, 99964L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854675843__9223370311919910408__31__9223372036854771234__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(1L, 4L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775803__9223370311919910413__31__9223372036854775806__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(2474L, 3462L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854772345__9223370311919910429__31__9223372036854773333__1", + translogTransferManager + ) + ); + assertEquals( + new Tuple<>(5807L, 7917L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854767890__9223370311919910434__31__9223372036854770000__1", + translogTransferManager + ) + ); + + // For older md filenames, it needs to read the content + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getMinTranslogGeneration()).thenReturn(701L); + when(md1.getGeneration()).thenReturn(1008L); + when(translogTransferManager.readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1")) + .thenReturn(md1); + assertEquals( + new Tuple<>(701L, 1008L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1", + translogTransferManager + ) + ); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); + when(md2.getMinTranslogGeneration()).thenReturn(4L); + when(md2.getGeneration()).thenReturn(7L); + when(translogTransferManager.readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1")) + .thenReturn(md2); + assertEquals( + new Tuple<>(4L, 7L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", + translogTransferManager + ) + ); + + verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"); + verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"); + } +} diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index ffa6a0f456f36..2426a14c0c93b 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -23,6 +23,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.transfer.FileSnapshot.CheckpointFileSnapshot; @@ -51,7 +52,10 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.mockito.Mockito; @@ -753,4 +757,121 @@ private void assertTlogCkpDownloadStatsWithMetadata() { // Expect delay for both tlog and ckp file assertTrue(remoteTranslogTransferTracker.getTotalDownloadTimeInMillis() >= delayForBlobDownload); } + + public void testlistTranslogMetadataFilesAsync() throws Exception { + String tm1 = new TranslogTransferMetadata(1, 1, 1, 2).getFileName(); + String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName(); + String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName(); + doAnswer(invocation -> { + ActionListener> actionListener = invocation.getArgument(4); + List bmList = new LinkedList<>(); + bmList.add(new PlainBlobMetadata(tm1, 1)); + bmList.add(new PlainBlobMetadata(tm2, 1)); + bmList.add(new PlainBlobMetadata(tm3, 1)); + actionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrderAsync( + eq(ThreadPool.Names.REMOTE_PURGE), + any(BlobPath.class), + eq(TranslogTransferMetadata.METADATA_PREFIX), + anyInt(), + any(ActionListener.class) + ); + AtomicBoolean fetchCompleted = new AtomicBoolean(false); + translogTransferManager.listTranslogMetadataFilesAsync(new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + assertEquals(3, blobMetadata.size()); + assertEquals(blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()), List.of(tm1, tm2, tm3)); + fetchCompleted.set(true); + } + + @Override + public void onFailure(Exception e) { + fetchCompleted.set(true); + throw new RuntimeException(e); + } + }); + assertBusy(() -> assertTrue(fetchCompleted.get())); + } + + public void testReadMetadataForGivenTimestampNoFile() throws IOException { + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + List bmList = new LinkedList<>(); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); + + assertNull(translogTransferManager.readMetadata(1234L)); + assertNoDownloadStats(false); + } + + public void testReadMetadataForGivenTimestampNoMatchingFile() throws IOException { + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + String timestamp1 = RemoteStoreUtils.invertLong(2345L); + BlobMetadata bm1 = new PlainBlobMetadata("metadata__1__12__" + timestamp1 + "__node1__1", 1); + String timestamp2 = RemoteStoreUtils.invertLong(3456L); + BlobMetadata bm2 = new PlainBlobMetadata("metadata__1__12__" + timestamp2 + "__node1__1", 1); + List bmList = List.of(bm1, bm2); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); + + assertNull(translogTransferManager.readMetadata(1234L)); + assertNoDownloadStats(false); + } + + public void testReadMetadataForGivenTimestampFile() throws IOException { + AtomicReference mdFilename1 = new AtomicReference<>(); + String timestamp1 = RemoteStoreUtils.invertLong(2345L); + mdFilename1.set("metadata__1__12__" + timestamp1 + "__node1__1"); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + BlobMetadata bm1 = new PlainBlobMetadata(mdFilename1.get(), 1); + String timestamp2 = RemoteStoreUtils.invertLong(3456L); + BlobMetadata bm2 = new PlainBlobMetadata("metadata__1__12__" + timestamp2 + "__node1__1", 1); + List bmList = List.of(bm1, bm2); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); + + TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); + long delayForMdDownload = 1; + when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename1.get()))).thenAnswer(invocation -> { + Thread.sleep(delayForMdDownload); + return new ByteArrayInputStream(translogTransferManager.getMetadataBytes(metadata)); + }); + + assertEquals(metadata, translogTransferManager.readMetadata(3000L)); + + assertEquals(translogTransferManager.getMetadataBytes(metadata).length, remoteTranslogTransferTracker.getDownloadBytesSucceeded()); + assertTrue(remoteTranslogTransferTracker.getTotalDownloadTimeInMillis() >= delayForMdDownload); + } + + public void testReadMetadataForGivenTimestampException() throws IOException { + AtomicReference mdFilename1 = new AtomicReference<>(); + String timestamp1 = RemoteStoreUtils.invertLong(2345L); + mdFilename1.set("metadata__1__12__" + timestamp1 + "__node1__1"); + doAnswer(invocation -> { + LatchedActionListener> latchedActionListener = invocation.getArgument(3); + BlobMetadata bm1 = new PlainBlobMetadata(mdFilename1.get(), 1); + String timestamp2 = RemoteStoreUtils.invertLong(3456L); + BlobMetadata bm2 = new PlainBlobMetadata("metadata__1__12__" + timestamp2 + "__node1__1", 1); + List bmList = List.of(bm1, bm2); + latchedActionListener.onResponse(bmList); + return null; + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); + + when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename1.get()))).thenThrow(new IOException("Something went wrong")); + + assertThrows(IOException.class, () -> translogTransferManager.readMetadata(3000L)); + assertNoDownloadStats(true); + } } From 758c2aa367dd4b4abe6db38dbb0b3f55d8354c1c Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Mon, 2 Sep 2024 11:39:40 +0530 Subject: [PATCH 003/357] Fix flaky tests with hashed prefix path type snapshots (#15560) Signed-off-by: Ashish Singh --- .../CorruptedBlobStoreRepositoryIT.java | 18 +----------------- .../opensearch/snapshots/RepositoriesIT.java | 9 ++++++++- .../snapshots/SnapshotStatusApisIT.java | 8 ++++---- .../test/OpenSearchIntegTestCase.java | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java index 95cc9eca92355..aeca4bb82e40c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -40,13 +40,9 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; -import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.unit.ByteSizeUnit; -import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; -import org.opensearch.index.remote.RemoteStoreEnums.PathType; -import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -62,7 +58,7 @@ import java.util.Map; import java.util.stream.Stream; -import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; +import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows; import static org.hamcrest.Matchers.containsString; @@ -683,16 +679,4 @@ private void assertRepositoryBlocked(Client client, String repo, String existing containsString("Could not read repository data because the contents of the repository do not match its expected state.") ); } - - private static String resolvePath(IndexId indexId, String shardId) { - PathType pathType = PathType.fromCode(indexId.getShardPathType()); - RemoteStorePathStrategy.SnapshotShardPathInput shardPathInput = new RemoteStorePathStrategy.SnapshotShardPathInput.Builder() - .basePath(BlobPath.cleanPath()) - .indexUUID(indexId.getId()) - .shardId(shardId) - .build(); - PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null; - BlobPath blobPath = pathType.path(shardPathInput, pathHashAlgorithm); - return blobPath.buildAsString(); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java index 97c09fc379bc8..271fcf166139f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java @@ -110,10 +110,17 @@ public void testRepositoryCreation() throws Exception { assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-1"), notNullValue()); assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue()); + RepositoryMetadata testRepo1Md = findRepository(repositoriesResponse.repositories(), "test-repo-1"); + logger.info("--> check that trying to create a repository with the same settings repeatedly does not update cluster state"); String beforeStateUuid = clusterStateResponse.getState().stateUUID(); createRepository("test-repo-1", "fs", Settings.builder().put("location", location)); - assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID()); + repositoriesResponse = client.admin().cluster().prepareGetRepositories(randomFrom("_all", "*", "test-repo-*")).get(); + RepositoryMetadata testRepo1MdAfterUpdate = findRepository(repositoriesResponse.repositories(), "test-repo-1"); + + if (testRepo1Md.settings().equals(testRepo1MdAfterUpdate.settings())) { + assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID()); + } logger.info("--> delete repository test-repo-1"); client.admin().cluster().prepareDeleteRepository("test-repo-1").get(); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index fb69209f7adda..3c75b30580c30 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -49,6 +49,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.repositories.IndexId; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -62,6 +63,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -200,11 +202,9 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap"); logger.info("--> delete shard-level snap-${uuid}.dat file for one shard in this snapshot to simulate concurrent delete"); - final String indexRepoId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0)).getId(); + IndexId indexId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0)); IOUtils.rm( - repoPath.resolve("indices") - .resolve(indexRepoId) - .resolve("0") + repoPath.resolve(resolvePath(indexId, "0")) .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat") ); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index de7dde5e9c6e9..f277c118c3d46 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -96,6 +96,7 @@ import org.opensearch.cluster.service.applicationtemplates.TestSystemTemplatesRepositoryPlugin; import org.opensearch.common.Nullable; import org.opensearch.common.Priority; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.collect.Tuple; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.network.NetworkModule; @@ -140,7 +141,9 @@ import org.opensearch.index.engine.Segment; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.MockFieldFilterPlugin; +import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.index.remote.RemoteStoreEnums.PathType; +import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; @@ -155,6 +158,7 @@ import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.IndexId; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.repositories.fs.ReloadableFsRepository; @@ -220,6 +224,7 @@ import static org.opensearch.index.IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING; import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -2884,4 +2889,16 @@ private static Settings buildRemoteStoreNodeAttributes( settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); return settings.build(); } + + public static String resolvePath(IndexId indexId, String shardId) { + PathType pathType = PathType.fromCode(indexId.getShardPathType()); + RemoteStorePathStrategy.SnapshotShardPathInput shardPathInput = new RemoteStorePathStrategy.SnapshotShardPathInput.Builder() + .basePath(BlobPath.cleanPath()) + .indexUUID(indexId.getId()) + .shardId(shardId) + .build(); + RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null; + BlobPath blobPath = pathType.path(shardPathInput, pathHashAlgorithm); + return blobPath.buildAsString(); + } } From c37d5f691fabb470181b7333efce5bbe4b058107 Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Mon, 2 Sep 2024 15:12:18 +0530 Subject: [PATCH 004/357] Modify create snapshot response when wait_for_completion is false (#15499) Signed-off-by: Anshu Agarwal Co-authored-by: Anshu Agarwal --- .../remotestore/RemoteRestoreSnapshotIT.java | 20 +++++++++---------- .../create/TransportCreateSnapshotAction.java | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 455a57071b287..56078a6ef8800 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -839,18 +839,13 @@ public void testCreateSnapshotV2() throws Exception { String snapshotName2 = "test-create-snapshot2"; - // verify even if waitForCompletion is not true, the request executes in a sync manner - CreateSnapshotResponse createSnapshotResponse2 = client().admin() + // verify response status if waitForCompletion is not true + RestStatus createSnapshotResponseStatus = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .get(); - snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); - assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); - + .get() + .status(); + assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus); } public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception { @@ -914,6 +909,7 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) .get(); snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); @@ -968,6 +964,7 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName) + .setWaitForCompletion(true) .get(); SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); @@ -1036,6 +1033,7 @@ public void testCreateSnapshotV2WithRedIndex() throws Exception { CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) .get(); SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); @@ -1097,6 +1095,7 @@ public void testCreateSnapshotV2WithIndexingLoad() throws Exception { CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) .get(); SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); @@ -1203,6 +1202,7 @@ public void testClusterManagerFailoverDuringSnapshotCreation() throws Exception CreateSnapshotResponse createSnapshotResponse = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) .get(); snapshotInfo[0] = createSnapshotResponse.getSnapshotInfo(); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 25e71d5598a98..6b582396c2733 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -113,8 +113,10 @@ protected void clusterManagerOperation( ) { Repository repository = repositoriesService.repository(request.repository()); boolean isSnapshotV2 = SHALLOW_SNAPSHOT_V2.get(repository.getMetadata().settings()); - if (request.waitForCompletion() || isSnapshotV2) { + if (request.waitForCompletion()) { snapshotsService.executeSnapshot(request, ActionListener.map(listener, CreateSnapshotResponse::new)); + } else if (isSnapshotV2) { + snapshotsService.executeSnapshot(request, ActionListener.map(listener, snapshot -> new CreateSnapshotResponse())); } else { snapshotsService.createSnapshot(request, ActionListener.map(listener, snapshot -> new CreateSnapshotResponse())); } From b54e867da0e313513e90872e039717b7595cf6e4 Mon Sep 17 00:00:00 2001 From: Arpit-Bandejiya Date: Mon, 2 Sep 2024 15:51:53 +0530 Subject: [PATCH 005/357] [Remote Publication] Add remote download stats (#15291) --------- Signed-off-by: Arpit Bandejiya --- CHANGELOG.md | 1 + .../RemoteClusterStateCleanupManagerIT.java | 14 +- .../remote/RemoteStatePublicationIT.java | 35 +++++ .../cluster/coordination/Coordinator.java | 4 + .../PublicationTransportHandler.java | 139 ++++++++++-------- .../opensearch/gateway/GatewayMetaState.java | 2 +- .../RemoteClusterStateCleanupManager.java | 2 +- .../remote/RemoteClusterStateService.java | 56 +++++-- .../remote/RemotePersistenceStats.java | 85 ++++++++--- .../gateway/remote/RemoteUploadStats.java | 59 ++++++++ .../cluster/node/stats/NodeStatsTests.java | 2 +- .../PublicationTransportHandlerTests.java | 93 +++++++++++- .../GatewayMetaStatePersistedStateTests.java | 17 ++- ...RemoteClusterStateCleanupManagerTests.java | 4 +- .../RemoteClusterStateServiceTests.java | 18 ++- 15 files changed, 412 insertions(+), 119 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteUploadStats.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 453f32ee74878..67091eaa6b850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410)) - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) +- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index 7d2e24c777da3..47ec3f25bcd64 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -31,6 +31,7 @@ import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -40,6 +41,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteUploadStats.REMOTE_UPLOAD; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -253,11 +255,13 @@ private void verifyIndexRoutingFilesDeletion( DiscoveryStats discoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats(); assertNotNull(discoveryStats.getClusterStateStats()); for (PersistedStateStats persistedStateStats : discoveryStats.getClusterStateStats().getPersistenceStats()) { - Map extendedFields = persistedStateStats.getExtendedFields(); - assertTrue(extendedFields.containsKey(RemotePersistenceStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT)); - long cleanupAttemptFailedCount = extendedFields.get(RemotePersistenceStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT) - .get(); - assertEquals(0, cleanupAttemptFailedCount); + if (Objects.equals(persistedStateStats.getStatsName(), REMOTE_UPLOAD)) { + Map extendedFields = persistedStateStats.getExtendedFields(); + assertTrue(extendedFields.containsKey(RemoteUploadStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT)); + long cleanupAttemptFailedCount = extendedFields.get(RemoteUploadStats.INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT) + .get(); + assertEquals(0, cleanupAttemptFailedCount); + } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 6a2e7ce4957ae..f237a81942e1a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -8,12 +8,15 @@ package org.opensearch.gateway.remote; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.client.Client; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.discovery.DiscoveryStats; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; @@ -155,6 +158,38 @@ public void testRemotePublicationDisableIfRemoteStateDisabled() { assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class)); } + public void testRemotePublicationDownloadStats() { + int shardCount = randomIntBetween(1, 2); + int replicaCount = 1; + int dataNodeCount = shardCount * (replicaCount + 1); + int clusterManagerNodeCount = 1; + prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount); + String dataNode = internalCluster().getDataNodeNames().stream().collect(Collectors.toList()).get(0); + + NodesStatsResponse nodesStatsResponseDataNode = client().admin() + .cluster() + .prepareNodesStats(dataNode) + .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName()) + .get(); + + assertDataNodeDownloadStats(nodesStatsResponseDataNode); + + } + + private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) { + // assert cluster state stats for data node + DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats(); + assertNotNull(dataNodeDiscoveryStats.getClusterStateStats()); + assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getUpdateSuccess()); + assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0); + assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount()); + assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0); + + assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getSuccessCount() > 0); + assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getFailedCount()); + assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getTotalTimeInMillis() > 0); + } + private Map getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException { BlobPath metadataPath = repository.basePath() .add( diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 306d28e055767..9aaaa77bcbb23 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -901,6 +901,10 @@ public DiscoveryStats stats() { stats.add(persistedStateRegistry.getPersistedState(stateType).getStats()); } }); + if (coordinationState.get().isRemotePublicationEnabled()) { + stats.add(publicationHandler.getFullDownloadStats()); + stats.add(publicationHandler.getDiffDownloadStats()); + } clusterStateStats.setPersistenceStats(stats); return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats(), clusterStateStats); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index f7e4eff655e9d..ca36011b3a0e9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -178,6 +178,14 @@ public PublishClusterStateStats stats() { ); } + public PersistedStateStats getFullDownloadStats() { + return remoteClusterStateService.getFullDownloadStats(); + } + + public PersistedStateStats getDiffDownloadStats() { + return remoteClusterStateService.getDiffDownloadStats(); + } + private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException { try (StreamInput in = CompressedStreamUtils.decompressBytes(request, namedWriteableRegistry)) { ClusterState incomingState; @@ -231,69 +239,78 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque } // package private for testing - PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException { - if (transportService.getLocalNode().equals(request.getSourceNode())) { - return acceptRemoteStateOnLocalNode(request); - } - // TODO Make cluster state download non-blocking: https://github.com/opensearch-project/OpenSearch/issues/14102 - ClusterMetadataManifest manifest = remoteClusterStateService.getClusterMetadataManifestByFileName( - request.getClusterUUID(), - request.getManifestFile() - ); - if (manifest == null) { - throw new IllegalStateException("Publication failed as manifest was not found for " + request); - } + PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException, IllegalStateException { boolean applyFullState = false; - final ClusterState lastSeen = lastSeenClusterState.get(); - if (lastSeen == null) { - logger.debug(() -> "Diff cannot be applied as there is no last cluster state"); - applyFullState = true; - } else if (manifest.getDiffManifest() == null) { - logger.trace(() -> "There is no diff in the manifest"); - applyFullState = true; - } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { - logger.debug(() -> "Last cluster state not compatible with the diff"); - applyFullState = true; - } - - if (applyFullState == true) { - logger.debug( - () -> new ParameterizedMessage( - "Downloading full cluster state for term {}, version {}, stateUUID {}", - manifest.getClusterTerm(), - manifest.getStateVersion(), - manifest.getStateUUID() - ) - ); - ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest( - request.getClusterName(), - manifest, - transportService.getLocalNode().getId(), - true - ); - fullClusterStateReceivedCount.incrementAndGet(); - final PublishWithJoinResponse response = acceptState(clusterState); - lastSeenClusterState.set(clusterState); - return response; - } else { - logger.debug( - () -> new ParameterizedMessage( - "Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", - manifest.getClusterTerm(), - manifest.getStateVersion(), - manifest.getDiffManifest().getFromStateUUID(), - manifest.getStateUUID() - ) - ); - ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff( - manifest, - lastSeen, - transportService.getLocalNode().getId() + try { + if (transportService.getLocalNode().equals(request.getSourceNode())) { + return acceptRemoteStateOnLocalNode(request); + } + // TODO Make cluster state download non-blocking: https://github.com/opensearch-project/OpenSearch/issues/14102 + ClusterMetadataManifest manifest = remoteClusterStateService.getClusterMetadataManifestByFileName( + request.getClusterUUID(), + request.getManifestFile() ); - compatibleClusterStateDiffReceivedCount.incrementAndGet(); - final PublishWithJoinResponse response = acceptState(clusterState); - lastSeenClusterState.compareAndSet(lastSeen, clusterState); - return response; + if (manifest == null) { + throw new IllegalStateException("Publication failed as manifest was not found for " + request); + } + final ClusterState lastSeen = lastSeenClusterState.get(); + if (lastSeen == null) { + logger.debug(() -> "Diff cannot be applied as there is no last cluster state"); + applyFullState = true; + } else if (manifest.getDiffManifest() == null) { + logger.debug(() -> "There is no diff in the manifest"); + applyFullState = true; + } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { + logger.debug(() -> "Last cluster state not compatible with the diff"); + applyFullState = true; + } + + if (applyFullState == true) { + logger.debug( + () -> new ParameterizedMessage( + "Downloading full cluster state for term {}, version {}, stateUUID {}", + manifest.getClusterTerm(), + manifest.getStateVersion(), + manifest.getStateUUID() + ) + ); + ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest( + request.getClusterName(), + manifest, + transportService.getLocalNode().getId(), + true + ); + fullClusterStateReceivedCount.incrementAndGet(); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.set(clusterState); + return response; + } else { + logger.debug( + () -> new ParameterizedMessage( + "Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", + manifest.getClusterTerm(), + manifest.getStateVersion(), + manifest.getDiffManifest().getFromStateUUID(), + manifest.getStateUUID() + ) + ); + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff( + manifest, + lastSeen, + transportService.getLocalNode().getId() + ); + compatibleClusterStateDiffReceivedCount.incrementAndGet(); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.compareAndSet(lastSeen, clusterState); + return response; + } + } catch (Exception e) { + if (applyFullState) { + remoteClusterStateService.fullDownloadFailed(); + } else { + remoteClusterStateService.diffDownloadFailed(); + } + throw e; } } diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index 108fcc13df2af..bd56c9e1757c6 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -746,7 +746,7 @@ assert verifyManifestAndClusterState(manifestDetails.getClusterMetadataManifest( @Override public PersistedStateStats getStats() { - return remoteClusterStateService.getStats(); + return remoteClusterStateService.getUploadStats(); } private boolean verifyManifestAndClusterState(ClusterMetadataManifest manifest, ClusterState clusterState) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 02db15477ff95..d9bd9669f138c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -81,7 +81,7 @@ public RemoteClusterStateCleanupManager( RemoteRoutingTableService remoteRoutingTableService ) { this.remoteClusterStateService = remoteClusterStateService; - this.remoteStateStats = remoteClusterStateService.getStats(); + this.remoteStateStats = remoteClusterStateService.getRemoteStateStats(); ClusterSettings clusterSettings = clusterService.getClusterSettings(); this.clusterApplierService = clusterService.getClusterApplierService(); this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 05be827e65dc3..2b3913ced0144 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.coordination.PersistedStateStats; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; @@ -258,8 +259,8 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); - remoteStateStats.stateSucceeded(); - remoteStateStats.stateTook(durationMillis); + remoteStateStats.stateUploadSucceeded(); + remoteStateStats.stateUploadTook(durationMillis); if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " @@ -450,8 +451,8 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); - remoteStateStats.stateSucceeded(); - remoteStateStats.stateTook(durationMillis); + remoteStateStats.stateUploadSucceeded(); + remoteStateStats.stateUploadTook(durationMillis); ParameterizedMessage clusterStateUploadTimeMessage = new ParameterizedMessage( CLUSTER_STATE_UPLOAD_TIME_LOG_STRING, manifestDetails.getClusterMetadataManifest().getStateVersion(), @@ -1315,8 +1316,10 @@ public ClusterState getClusterStateForManifest( String localNodeId, boolean includeEphemeral ) throws IOException { + final ClusterState clusterState; + final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (manifest.onOrAfterCodecVersion(CODEC_V2)) { - return readClusterStateInParallel( + clusterState = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, manifest.getClusterUUID(), @@ -1336,7 +1339,7 @@ public ClusterState getClusterStateForManifest( includeEphemeral ); } else { - ClusterState clusterState = readClusterStateInParallel( + ClusterState state = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, manifest.getClusterUUID(), @@ -1357,15 +1360,20 @@ public ClusterState getClusterStateForManifest( false ); Metadata.Builder mb = Metadata.builder(remoteGlobalMetadataManager.getGlobalMetadata(manifest.getClusterUUID(), manifest)); - mb.indices(clusterState.metadata().indices()); - return ClusterState.builder(clusterState).metadata(mb).build(); + mb.indices(state.metadata().indices()); + clusterState = ClusterState.builder(state).metadata(mb).build(); } + final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); + remoteStateStats.stateFullDownloadSucceeded(); + remoteStateStats.stateFullDownloadTook(durationMillis); + return clusterState; } public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) throws IOException { assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; + final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); ClusterStateDiffManifest diff = manifest.getDiffManifest(); List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { Optional uploadedIndexMetadataOptional = manifest.getIndices() @@ -1441,11 +1449,17 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C indexRoutingTables.remove(indexName); } - return clusterStateBuilder.stateUUID(manifest.getStateUUID()) + ClusterState clusterState = clusterStateBuilder.stateUUID(manifest.getStateUUID()) .version(manifest.getStateVersion()) .metadata(metadataBuilder) .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); + + final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); + remoteStateStats.stateDiffDownloadSucceeded(); + remoteStateStats.stateDiffDownloadTook(durationMillis); + + return clusterState; } /** @@ -1641,10 +1655,30 @@ void setRemoteClusterStateAttributesManager(RemoteClusterStateAttributesManager } public void writeMetadataFailed() { - getStats().stateFailed(); + remoteStateStats.stateUploadFailed(); } - public RemotePersistenceStats getStats() { + public RemotePersistenceStats getRemoteStateStats() { return remoteStateStats; } + + public PersistedStateStats getUploadStats() { + return remoteStateStats.getUploadStats(); + } + + public PersistedStateStats getFullDownloadStats() { + return remoteStateStats.getRemoteFullDownloadStats(); + } + + public PersistedStateStats getDiffDownloadStats() { + return remoteStateStats.getRemoteDiffDownloadStats(); + } + + public void fullDownloadFailed() { + remoteStateStats.stateFullDownloadFailed(); + } + + public void diffDownloadFailed() { + remoteStateStats.stateDiffDownloadFailed(); + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 1e7f8f278fb0f..417ebdafd3ba7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -10,51 +10,96 @@ import org.opensearch.cluster.coordination.PersistedStateStats; -import java.util.concurrent.atomic.AtomicLong; - /** * Remote state related extended stats. * * @opensearch.internal */ -public class RemotePersistenceStats extends PersistedStateStats { - static final String CLEANUP_ATTEMPT_FAILED_COUNT = "cleanup_attempt_failed_count"; - static final String INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT = "index_routing_files_cleanup_attempt_failed_count"; - static final String INDICES_ROUTING_DIFF_FILES_CLEANUP_ATTEMPT_FAILED_COUNT = "indices_routing_diff_files_cleanup_attempt_failed_count"; - static final String REMOTE_UPLOAD = "remote_upload"; - private AtomicLong cleanupAttemptFailedCount = new AtomicLong(0); +public class RemotePersistenceStats { + + RemoteUploadStats remoteUploadStats; + PersistedStateStats remoteDiffDownloadStats; + PersistedStateStats remoteFullDownloadStats; - private AtomicLong indexRoutingFilesCleanupAttemptFailedCount = new AtomicLong(0); - private AtomicLong indicesRoutingDiffFilesCleanupAttemptFailedCount = new AtomicLong(0); + final String FULL_DOWNLOAD_STATS = "remote_full_download"; + final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; public RemotePersistenceStats() { - super(REMOTE_UPLOAD); - addToExtendedFields(CLEANUP_ATTEMPT_FAILED_COUNT, cleanupAttemptFailedCount); - addToExtendedFields(INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT, indexRoutingFilesCleanupAttemptFailedCount); - addToExtendedFields(INDICES_ROUTING_DIFF_FILES_CLEANUP_ATTEMPT_FAILED_COUNT, indicesRoutingDiffFilesCleanupAttemptFailedCount); + remoteUploadStats = new RemoteUploadStats(); + remoteDiffDownloadStats = new PersistedStateStats(DIFF_DOWNLOAD_STATS); + remoteFullDownloadStats = new PersistedStateStats(FULL_DOWNLOAD_STATS); } public void cleanUpAttemptFailed() { - cleanupAttemptFailedCount.incrementAndGet(); + remoteUploadStats.cleanUpAttemptFailed(); } public long getCleanupAttemptFailedCount() { - return cleanupAttemptFailedCount.get(); + return remoteUploadStats.getCleanupAttemptFailedCount(); } public void indexRoutingFilesCleanupAttemptFailed() { - indexRoutingFilesCleanupAttemptFailedCount.incrementAndGet(); + remoteUploadStats.indexRoutingFilesCleanupAttemptFailed(); } public long getIndexRoutingFilesCleanupAttemptFailedCount() { - return indexRoutingFilesCleanupAttemptFailedCount.get(); + return remoteUploadStats.getIndexRoutingFilesCleanupAttemptFailedCount(); } public void indicesRoutingDiffFileCleanupAttemptFailed() { - indicesRoutingDiffFilesCleanupAttemptFailedCount.incrementAndGet(); + remoteUploadStats.indicesRoutingDiffFileCleanupAttemptFailed(); } public long getIndicesRoutingDiffFileCleanupAttemptFailedCount() { - return indicesRoutingDiffFilesCleanupAttemptFailedCount.get(); + return remoteUploadStats.getIndicesRoutingDiffFileCleanupAttemptFailedCount(); + } + + public void stateUploadSucceeded() { + remoteUploadStats.stateSucceeded(); + } + + public void stateUploadTook(long durationMillis) { + remoteUploadStats.stateTook(durationMillis); + } + + public void stateUploadFailed() { + remoteUploadStats.stateFailed(); + } + + public void stateFullDownloadSucceeded() { + remoteFullDownloadStats.stateSucceeded(); + } + + public void stateDiffDownloadSucceeded() { + remoteDiffDownloadStats.stateSucceeded(); + } + + public void stateFullDownloadTook(long durationMillis) { + remoteFullDownloadStats.stateTook(durationMillis); } + + public void stateDiffDownloadTook(long durationMillis) { + remoteDiffDownloadStats.stateTook(durationMillis); + } + + public void stateFullDownloadFailed() { + remoteFullDownloadStats.stateFailed(); + } + + public void stateDiffDownloadFailed() { + remoteDiffDownloadStats.stateFailed(); + } + + public PersistedStateStats getUploadStats() { + return remoteUploadStats; + } + + public PersistedStateStats getRemoteDiffDownloadStats() { + return remoteDiffDownloadStats; + } + + public PersistedStateStats getRemoteFullDownloadStats() { + return remoteFullDownloadStats; + } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteUploadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteUploadStats.java new file mode 100644 index 0000000000000..9ffef65ae1eba --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteUploadStats.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.coordination.PersistedStateStats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Upload stats for remote state + * + * @opensearch.internal + */ +public class RemoteUploadStats extends PersistedStateStats { + static final String CLEANUP_ATTEMPT_FAILED_COUNT = "cleanup_attempt_failed_count"; + static final String INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT = "index_routing_files_cleanup_attempt_failed_count"; + static final String INDICES_ROUTING_DIFF_FILES_CLEANUP_ATTEMPT_FAILED_COUNT = "indices_routing_diff_files_cleanup_attempt_failed_count"; + static final String REMOTE_UPLOAD = "remote_upload"; + private AtomicLong cleanupAttemptFailedCount = new AtomicLong(0); + private AtomicLong indexRoutingFilesCleanupAttemptFailedCount = new AtomicLong(0); + private AtomicLong indicesRoutingDiffFilesCleanupAttemptFailedCount = new AtomicLong(0); + + public RemoteUploadStats() { + super(REMOTE_UPLOAD); + addToExtendedFields(CLEANUP_ATTEMPT_FAILED_COUNT, cleanupAttemptFailedCount); + addToExtendedFields(INDEX_ROUTING_FILES_CLEANUP_ATTEMPT_FAILED_COUNT, indexRoutingFilesCleanupAttemptFailedCount); + addToExtendedFields(INDICES_ROUTING_DIFF_FILES_CLEANUP_ATTEMPT_FAILED_COUNT, indicesRoutingDiffFilesCleanupAttemptFailedCount); + } + + public void cleanUpAttemptFailed() { + cleanupAttemptFailedCount.incrementAndGet(); + } + + public long getCleanupAttemptFailedCount() { + return cleanupAttemptFailedCount.get(); + } + + public void indexRoutingFilesCleanupAttemptFailed() { + indexRoutingFilesCleanupAttemptFailedCount.incrementAndGet(); + } + + public long getIndexRoutingFilesCleanupAttemptFailedCount() { + return indexRoutingFilesCleanupAttemptFailedCount.get(); + } + + public void indicesRoutingDiffFileCleanupAttemptFailed() { + indicesRoutingDiffFilesCleanupAttemptFailedCount.incrementAndGet(); + } + + public long getIndicesRoutingDiffFileCleanupAttemptFailedCount() { + return indicesRoutingDiffFilesCleanupAttemptFailedCount.get(); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index a0225a0bf6193..11902728eed07 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -823,7 +823,7 @@ public static NodeStats createNodeStats(boolean remoteStoreStats) throws IOExcep : null; ClusterStateStats stateStats = new ClusterStateStats(); RemotePersistenceStats remoteStateStats = new RemotePersistenceStats(); - stateStats.setPersistenceStats(Arrays.asList(remoteStateStats)); + stateStats.setPersistenceStats(Arrays.asList(remoteStateStats.getUploadStats())); DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats( randomBoolean() ? new PendingClusterStateStats(randomInt(), randomInt(), randomInt()) : null, diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 65bc228cd5704..266928c919fe2 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -50,6 +50,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.GatewayMetaState.RemotePersistedState; import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.ClusterStateDiffManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.telemetry.tracing.noop.NoopTracer; @@ -74,8 +75,12 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; public class PublicationTransportHandlerTests extends OpenSearchTestCase { @@ -175,7 +180,9 @@ public void testHandleIncomingRemotePublishRequestWhenNoCurrentPublishRequest() () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - Mockito.verifyNoInteractions(remoteClusterStateService); + verify(remoteClusterStateService, times(0)).fullDownloadFailed(); + verify(remoteClusterStateService, times(1)).diffDownloadFailed(); + verifyNoMoreInteractions(remoteClusterStateService); } public void testHandleIncomingRemotePublishRequestWhenTermMismatch() { @@ -200,7 +207,9 @@ public void testHandleIncomingRemotePublishRequestWhenTermMismatch() { () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - Mockito.verifyNoInteractions(remoteClusterStateService); + verify(remoteClusterStateService, times(0)).fullDownloadFailed(); + verify(remoteClusterStateService, times(1)).diffDownloadFailed(); + verifyNoMoreInteractions(remoteClusterStateService); } public void testHandleIncomingRemotePublishRequestWhenVersionMismatch() { @@ -225,7 +234,9 @@ public void testHandleIncomingRemotePublishRequestWhenVersionMismatch() { () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - Mockito.verifyNoInteractions(remoteClusterStateService); + verify(remoteClusterStateService, times(1)).diffDownloadFailed(); + verify(remoteClusterStateService, times(0)).fullDownloadFailed(); + verifyNoMoreInteractions(remoteClusterStateService); } public void testHandleIncomingRemotePublishRequestForLocalNode() throws IOException { @@ -250,6 +261,82 @@ public void testHandleIncomingRemotePublishRequestForLocalNode() throws IOExcept Mockito.verifyNoInteractions(remoteClusterStateService); } + public void testDownloadRemotePersistedFullStateFailedStats() throws IOException { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + PersistedStateStats remoteFullDownloadStats = new PersistedStateStats("dummy_full_stats"); + PersistedStateStats remoteDiffDownloadStats = new PersistedStateStats("dummy_diff_stats"); + when(remoteClusterStateService.getFullDownloadStats()).thenReturn(remoteFullDownloadStats); + when(remoteClusterStateService.getDiffDownloadStats()).thenReturn(remoteDiffDownloadStats); + + doAnswer((i) -> { + remoteFullDownloadStats.stateFailed(); + return null; + }).when(remoteClusterStateService).fullDownloadFailed(); + + doAnswer((i) -> { + remoteDiffDownloadStats.stateFailed(); + return null; + }).when(remoteClusterStateService).diffDownloadFailed(); + + PublishWithJoinResponse expectedPublishResponse = new PublishWithJoinResponse(new PublishResponse(TERM, VERSION), Optional.empty()); + Function handlePublishRequest = p -> expectedPublishResponse; + final PublicationTransportHandler handler = getPublicationTransportHandler(handlePublishRequest, remoteClusterStateService); + RemotePublishRequest remotePublishRequest = new RemotePublishRequest( + secondNode, + TERM, + VERSION, + CLUSTER_NAME, + CLUSTER_UUID, + MANIFEST_FILE + ); + ClusterState clusterState = buildClusterState(TERM, VERSION); + PublishRequest publishRequest = new PublishRequest(clusterState); + handler.setCurrentPublishRequestToSelf(publishRequest); + + assertThrows(IllegalStateException.class, () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest)); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); + assertEquals(0, remoteClusterStateService.getFullDownloadStats().getFailedCount()); + } + + public void testDownloadRemotePersistedDiffStateFailedStats() throws IOException { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + PersistedStateStats remoteDiffDownloadStats = new PersistedStateStats("dummy_stats"); + when(remoteClusterStateService.getDiffDownloadStats()).thenReturn(remoteDiffDownloadStats); + + ClusterMetadataManifest metadataManifest = new ClusterMetadataManifest.Builder().diffManifest( + new ClusterStateDiffManifest.Builder().fromStateUUID("state-uuid").build() + ).build(); + when(remoteClusterStateService.getClusterMetadataManifestByFileName(any(), any())).thenReturn(metadataManifest); + + doAnswer((i) -> { + remoteDiffDownloadStats.stateFailed(); + return null; + }).when(remoteClusterStateService).diffDownloadFailed(); + + PublishWithJoinResponse expectedPublishResponse = new PublishWithJoinResponse(new PublishResponse(TERM, VERSION), Optional.empty()); + Function handlePublishRequest = p -> expectedPublishResponse; + final PublicationTransportHandler handler = getPublicationTransportHandler(handlePublishRequest, remoteClusterStateService); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.nodes()).thenReturn(mock(DiscoveryNodes.class)); + handler.setLastSeenClusterState(clusterState); + when(clusterState.stateUUID()).thenReturn("state-uuid"); + RemotePublishRequest remotePublishRequest = new RemotePublishRequest( + secondNode, + TERM, + VERSION, + CLUSTER_NAME, + CLUSTER_UUID, + MANIFEST_FILE + ); + clusterState = buildClusterState(TERM, VERSION); + PublishRequest publishRequest = new PublishRequest(clusterState); + handler.setCurrentPublishRequestToSelf(publishRequest); + + assertThrows(NullPointerException.class, () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest)); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); + + } + public void testHandleIncomingRemotePublishRequestWhenManifestNotFound() throws IOException { RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 529302bea6758..5ac94281822b8 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -68,7 +68,7 @@ import org.opensearch.gateway.PersistedClusterStateService.Writer; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; -import org.opensearch.gateway.remote.RemotePersistenceStats; +import org.opensearch.gateway.remote.RemoteUploadStats; import org.opensearch.gateway.remote.model.RemoteClusterStateManifestInfo; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; @@ -112,7 +112,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -899,14 +899,17 @@ public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOExcept } public void testRemotePersistedStateFailureStats() throws IOException { - RemotePersistenceStats remoteStateStats = new RemotePersistenceStats(); + RemoteUploadStats remoteStateStats = new RemoteUploadStats(); final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final String previousClusterUUID = "prev-cluster-uuid"; Mockito.doThrow(IOException.class) .when(remoteClusterStateService) .writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION)); - when(remoteClusterStateService.getStats()).thenReturn(remoteStateStats); - doCallRealMethod().when(remoteClusterStateService).writeMetadataFailed(); + when(remoteClusterStateService.getUploadStats()).thenReturn(remoteStateStats); + doAnswer((i) -> { + remoteStateStats.stateFailed(); + return null; + }).when(remoteClusterStateService).writeMetadataFailed(); CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); final long clusterTerm = randomNonNegativeLong(); @@ -916,8 +919,8 @@ public void testRemotePersistedStateFailureStats() throws IOException { ); assertThrows(OpenSearchException.class, () -> remotePersistedState.setLastAcceptedState(clusterState)); - assertEquals(1, remoteClusterStateService.getStats().getFailedCount()); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); + assertEquals(1, remoteClusterStateService.getUploadStats().getFailedCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } public void testGatewayForRemoteState() throws IOException { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 920a48f02b99a..8e114c9a26534 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -144,7 +144,7 @@ public void setup() { remoteManifestManager = mock(RemoteManifestManager.class); remoteClusterStateService = mock(RemoteClusterStateService.class); when(remoteClusterStateService.getRemoteManifestManager()).thenReturn(remoteManifestManager); - when(remoteClusterStateService.getStats()).thenReturn(new RemotePersistenceStats()); + when(remoteClusterStateService.getRemoteStateStats()).thenReturn(new RemotePersistenceStats()); when(remoteClusterStateService.getThreadpool()).thenReturn(threadPool); when(remoteClusterStateService.getBlobStore()).thenReturn(blobStore); when(remoteClusterStateService.getBlobStoreRepository()).thenReturn(blobStoreRepository); @@ -503,7 +503,7 @@ public void testRemoteStateCleanupFailureStats() throws IOException { assertBusy(() -> { // wait for stats to get updated assertTrue(remoteClusterStateCleanupManager.getStats() != null); - assertEquals(0, remoteClusterStateCleanupManager.getStats().getSuccessCount()); + assertEquals(0, remoteClusterStateCleanupManager.getStats().getUploadStats().getSuccessCount()); assertEquals(1, remoteClusterStateCleanupManager.getStats().getCleanupAttemptFailedCount()); }); } catch (Exception e) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 168c74c7cb584..d57cc17acedcb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -575,7 +575,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx RemoteStateTransferException.class, () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) ); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOException { @@ -587,7 +587,7 @@ public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOExc null ); Assert.assertThat(manifestDetails, nullValue()); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } public void testFailWriteIncrementalMetadataWhenTermChanged() { @@ -861,6 +861,10 @@ public void testGetClusterStateForManifest_IncludeEphemeral() throws IOException when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA); RemoteClusterStateService mockService = spy(remoteClusterStateService); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + + assertNotNull(remoteClusterStateService.getFullDownloadStats()); + assertEquals(1, remoteClusterStateService.getFullDownloadStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getFullDownloadStats().getFailedCount()); verify(mockService, times(1)).readClusterStateInParallel( any(), eq(manifest), @@ -2590,7 +2594,7 @@ public void testGetValidPreviousClusterUUIDWhenLastUUIDUncommitted() throws IOEx assertThat(previousClusterUUID, equalTo("cluster-uuid2")); } - public void testRemoteStateStats() throws IOException { + public void testRemoteStateUploadStats() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); @@ -2600,10 +2604,10 @@ public void testRemoteStateStats() throws IOException { MANIFEST_CURRENT_CODEC_VERSION ).getClusterMetadataManifest(); - assertTrue(remoteClusterStateService.getStats() != null); - assertEquals(1, remoteClusterStateService.getStats().getSuccessCount()); - assertEquals(0, remoteClusterStateService.getStats().getCleanupAttemptFailedCount()); - assertEquals(0, remoteClusterStateService.getStats().getFailedCount()); + assertTrue(remoteClusterStateService.getUploadStats() != null); + assertEquals(1, remoteClusterStateService.getUploadStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getCleanupAttemptFailedCount()); + assertEquals(0, remoteClusterStateService.getUploadStats().getFailedCount()); } public void testRemoteRoutingTableNotInitializedWhenDisabled() { From 6bae7045702b60c736cc9899224c8652e5803968 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 2 Sep 2024 17:51:43 +0530 Subject: [PATCH 006/357] Star Tree File Formats (#14809) --------- Signed-off-by: Sarthak Aggarwal --- .../codec/composite/CompositeIndexReader.java | 1 + .../Composite99DocValuesFormat.java | 30 + .../Composite99DocValuesReader.java | 231 +- .../Composite99DocValuesWriter.java | 137 +- .../datacube/startree/StarTreeValues.java | 70 - .../CompositeIndexConstants.java | 5 + .../startree/builder/BaseStarTreeBuilder.java | 516 +++-- .../builder/OffHeapStarTreeBuilder.java | 48 +- .../builder/OnHeapStarTreeBuilder.java | 54 +- .../startree/builder/StarTreeBuilder.java | 24 +- .../startree/builder/StarTreesBuilder.java | 55 +- .../startree/fileformats/StarTreeWriter.java | 7 +- .../fileformats/data/StarTreeDataWriter.java | 38 +- .../fileformats/meta/StarTreeMetadata.java | 32 +- ...riter.java => StarTreeMetadataWriter.java} | 24 +- .../node/FixedLengthStarTreeNode.java | 7 +- .../startree/index}/CompositeIndexValues.java | 2 +- .../startree/index/StarTreeValues.java | 265 +++ .../startree/index}/package-info.java | 4 +- .../startree/node/InMemoryTreeNode.java | 114 +- .../startree/node/StarTreeNodeType.java | 18 +- .../datacube/startree/node/package-info.java | 2 + .../utils/SequentialDocValuesIterator.java | 18 +- .../startree/utils/StarTreeUtils.java | 40 +- .../index/engine/SegmentsStats.java | 5 + .../StarTreeDocValuesFormatTests.java | 69 +- .../datacube/startree/StarTreeTestUtils.java | 297 +++ .../builder/AbstractStarTreeBuilderTests.java | 1886 ++++++++++++++--- .../builder/BaseStarTreeBuilderTests.java | 38 +- .../builder/OffHeapStarTreeBuilderTests.java | 12 +- .../builder/OnHeapStarTreeBuilderTests.java | 17 +- .../builder/StarTreesBuilderTests.java | 29 +- .../data/StarTreeFileFormatsTests.java | 55 +- .../meta/StarTreeMetadataTests.java | 3 + .../node/FixedLengthStarTreeNodeTests.java | 125 +- .../startree/node/StarTreeNodeTypeTests.java | 14 +- .../startree/utils/StarTreeUtilsTests.java | 2 +- .../opensearch/test/OpenSearchTestCase.java | 8 + 38 files changed, 3479 insertions(+), 823 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/{StarTreeMetaWriter.java => StarTreeMetadataWriter.java} (90%) rename server/src/main/java/org/opensearch/index/{codec/composite => compositeindex/datacube/startree/index}/CompositeIndexValues.java (86%) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java rename server/src/main/java/org/opensearch/index/{codec/composite/datacube/startree => compositeindex/datacube/startree/index}/package-info.java (57%) rename server/src/test/java/org/opensearch/index/codec/{composite => composite99}/datacube/startree/StarTreeDocValuesFormatTests.java (63%) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java index a159b0619bcbb..9beb99e4a97cd 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java @@ -9,6 +9,7 @@ package org.opensearch.index.codec.composite; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java index e8c69b11b7c88..63fe5a6af76f7 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java @@ -37,6 +37,36 @@ public class Composite99DocValuesFormat extends DocValuesFormat { private final DocValuesFormat delegate; private final MapperService mapperService; + /** Data codec name for Composite Doc Values Format */ + public static final String DATA_CODEC_NAME = "Composite99FormatData"; + + /** Meta codec name for Composite Doc Values Format */ + public static final String META_CODEC_NAME = "Composite99FormatMeta"; + + /** Filename extension for the composite index data */ + public static final String DATA_EXTENSION = "cid"; + + /** Filename extension for the composite index meta */ + public static final String META_EXTENSION = "cim"; + + /** Data doc values codec name for Composite Doc Values Format */ + public static final String DATA_DOC_VALUES_CODEC = "Composite99DocValuesData"; + + /** Meta doc values codec name for Composite Doc Values Format */ + public static final String META_DOC_VALUES_CODEC = "Composite99DocValuesMetadata"; + + /** Filename extension for the composite index data doc values */ + public static final String DATA_DOC_VALUES_EXTENSION = "cidvd"; + + /** Filename extension for the composite index meta doc values */ + public static final String META_DOC_VALUES_EXTENSION = "cidvm"; + + /** Initial version for the Composite90DocValuesFormat */ + public static final int VERSION_START = 0; + + /** Current version for the Composite90DocValuesFormat */ + public static final int VERSION_CURRENT = VERSION_START; + // needed for SPI public Composite99DocValuesFormat() { this(new Lucene90DocValuesFormat(), null); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java index e3bfe01cfa2d5..ca55e8e4573ab 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java @@ -8,22 +8,47 @@ package org.opensearch.index.codec.composite.composite99; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; -import org.opensearch.index.codec.composite.CompositeIndexValues; +import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; +import org.opensearch.index.compositeindex.CompositeIndexMetadata; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.mapper.CompositeMappedFieldType; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.getFieldInfoList; /** * Reader for star tree index and star tree doc values from the segments @@ -32,11 +57,158 @@ */ @ExperimentalApi public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader { - private DocValuesProducer delegate; + private static final Logger logger = LogManager.getLogger(Composite99DocValuesReader.class); + + private final DocValuesProducer delegate; + private IndexInput dataIn; + private ChecksumIndexInput metaIn; + private final Map compositeIndexInputMap = new LinkedHashMap<>(); + private final Map compositeIndexMetadataMap = new LinkedHashMap<>(); + private final List fields; + private DocValuesProducer compositeDocValuesProducer; + private final List compositeFieldInfos = new ArrayList<>(); + private SegmentReadState readState; - public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException { + public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException { this.delegate = producer; - // TODO : read star tree files + this.fields = new ArrayList<>(); + + String metaFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Composite99DocValuesFormat.META_EXTENSION + ); + + String dataFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Composite99DocValuesFormat.DATA_EXTENSION + ); + + boolean success = false; + try { + + // initialize meta input + dataIn = readState.directory.openInput(dataFileName, readState.context); + CodecUtil.checkIndexHeader( + dataIn, + Composite99DocValuesFormat.DATA_CODEC_NAME, + Composite99DocValuesFormat.VERSION_START, + Composite99DocValuesFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + // initialize data input + metaIn = readState.directory.openChecksumInput(metaFileName, readState.context); + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader( + metaIn, + Composite99DocValuesFormat.META_CODEC_NAME, + Composite99DocValuesFormat.VERSION_START, + Composite99DocValuesFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + while (true) { + + // validate magic marker + long magicMarker = metaIn.readLong(); + if (magicMarker == -1) { + break; + } else if (magicMarker < 0) { + throw new CorruptIndexException("Unknown token encountered: " + magicMarker, metaIn); + } else if (COMPOSITE_FIELD_MARKER != magicMarker) { + logger.error("Invalid composite field magic marker"); + throw new IOException("Invalid composite field magic marker"); + } + + int version = metaIn.readVInt(); + if (VERSION_CURRENT != version) { + logger.error("Invalid composite field version"); + throw new IOException("Invalid composite field version"); + } + + // construct composite index metadata + String compositeFieldName = metaIn.readString(); + CompositeMappedFieldType.CompositeFieldType compositeFieldType = CompositeMappedFieldType.CompositeFieldType.fromName( + metaIn.readString() + ); + + switch (compositeFieldType) { + case STAR_TREE: + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + metaIn, + compositeFieldName, + compositeFieldType, + version + ); + compositeFieldInfos.add(new CompositeIndexFieldInfo(compositeFieldName, compositeFieldType)); + + IndexInput starTreeIndexInput = dataIn.slice( + "star-tree data slice for respective star-tree fields", + starTreeMetadata.getDataStartFilePointer(), + starTreeMetadata.getDataLength() + ); + compositeIndexInputMap.put(compositeFieldName, starTreeIndexInput); + compositeIndexMetadataMap.put(compositeFieldName, starTreeMetadata); + + List dimensionFields = starTreeMetadata.getDimensionFields(); + + // generating star tree unique fields (fully qualified name for dimension and metrics) + for (String dimensions : dimensionFields) { + fields.add(fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeFieldName, dimensions)); + } + + // adding metric fields + for (Metric metric : starTreeMetadata.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + fields.add( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeFieldName, + metric.getField(), + metricStat.getTypeName() + ) + ); + + } + } + + break; + default: + throw new CorruptIndexException("Invalid composite field type found in the file", dataIn); + } + } + + // populates the dummy list of field infos to fetch doc id set iterators for respective fields. + // the dummy field info is used to fetch the doc id set iterators for respective fields based on field name + FieldInfos fieldInfos = new FieldInfos(getFieldInfoList(fields)); + this.readState = new SegmentReadState(readState.directory, readState.segmentInfo, fieldInfos, readState.context); + + // initialize star-tree doc values producer + + compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( + Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + this.readState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + + } catch (Throwable t) { + priorE = t; + } finally { + CodecUtil.checkFooter(metaIn, priorE); + } + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } } @Override @@ -67,24 +239,63 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { @Override public void checkIntegrity() throws IOException { delegate.checkIntegrity(); - // Todo : check integrity of composite index related [star tree] files + CodecUtil.checksumEntireFile(dataIn); } @Override public void close() throws IOException { delegate.close(); - // Todo: close composite index related files [star tree] files + boolean success = false; + try { + IOUtils.close(metaIn, dataIn); + IOUtils.close(compositeDocValuesProducer); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(metaIn, dataIn); + } + compositeIndexInputMap.clear(); + compositeIndexMetadataMap.clear(); + fields.clear(); + metaIn = null; + dataIn = null; + } } @Override public List getCompositeIndexFields() { - // todo : read from file formats and get the field names. - return new ArrayList<>(); + return compositeFieldInfos; } @Override public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo compositeIndexFieldInfo) throws IOException { - // TODO : read compositeIndexValues [starTreeValues] from star tree files - throw new UnsupportedOperationException(); + + switch (compositeIndexFieldInfo.getType()) { + case STAR_TREE: + return new StarTreeValues( + compositeIndexMetadataMap.get(compositeIndexFieldInfo.getField()), + compositeIndexInputMap.get(compositeIndexFieldInfo.getField()), + compositeDocValuesProducer, + this.readState + ); + + default: + throw new CorruptIndexException("Unsupported composite index field type: ", compositeIndexFieldInfo.getType().getName()); + } + } + + /** + * Returns the sorted numeric doc values for the given sorted numeric field. + * If the sorted numeric field is null, it returns an empty doc id set iterator. + *

+ * Sorted numeric field can be null for cases where the segment doesn't hold a particular value. + * + * @param sortedNumeric the sorted numeric doc values for a field + * @return empty sorted numeric values if the field is not present, else sortedNumeric + */ + public static SortedNumericDocValues getSortedNumericDocValues(SortedNumericDocValues sortedNumeric) { + return sortedNumeric == null ? DocValues.emptySortedNumeric() : sortedNumeric; + } + } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java index 74ab7d423998e..0d4e35f7c3ab8 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java @@ -8,34 +8,44 @@ package org.opensearch.index.codec.composite.composite99; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; -import org.opensearch.index.codec.composite.CompositeIndexValues; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; +import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.StarTreeMapper; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; /** @@ -52,21 +62,28 @@ public class Composite99DocValuesWriter extends DocValuesConsumer { AtomicReference mergeState = new AtomicReference<>(); private final Set compositeMappedFieldTypes; private final Set compositeFieldSet; + private DocValuesConsumer composite99DocValuesConsumer; + + public IndexOutput dataOut; + public IndexOutput metaOut; private final Set segmentFieldSet; private final boolean segmentHasCompositeFields; + private final AtomicInteger fieldNumberAcrossCompositeFields; private final Map fieldProducerMap = new HashMap<>(); - public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) { + public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) + throws IOException { this.delegate = delegate; this.state = segmentWriteState; this.mapperService = mapperService; + this.fieldNumberAcrossCompositeFields = new AtomicInteger(); this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); compositeFieldSet = new HashSet<>(); segmentFieldSet = new HashSet<>(); // TODO : add integ test for this - for (FieldInfo fi : segmentWriteState.fieldInfos) { + for (FieldInfo fi : this.state.fieldInfos) { if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { segmentFieldSet.add(fi.name); } else if (fi.name.equals(DocCountFieldMapper.NAME)) { @@ -76,6 +93,56 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState for (CompositeMappedFieldType type : compositeMappedFieldTypes) { compositeFieldSet.addAll(type.fields()); } + + boolean success = false; + try { + + // Get consumer write state with DocIdSetIterator.NO_MORE_DOCS as segment doc count, + // so that all the fields are sparse numeric doc values and not dense numeric doc values + SegmentWriteState consumerWriteState = getSegmentWriteState(segmentWriteState); + + this.composite99DocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + consumerWriteState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + + String dataFileName = IndexFileNames.segmentFileName( + this.state.segmentInfo.name, + this.state.segmentSuffix, + Composite99DocValuesFormat.DATA_EXTENSION + ); + dataOut = this.state.directory.createOutput(dataFileName, this.state.context); + CodecUtil.writeIndexHeader( + dataOut, + Composite99DocValuesFormat.DATA_CODEC_NAME, + Composite99DocValuesFormat.VERSION_CURRENT, + this.state.segmentInfo.getId(), + this.state.segmentSuffix + ); + + String metaFileName = IndexFileNames.segmentFileName( + this.state.segmentInfo.name, + this.state.segmentSuffix, + Composite99DocValuesFormat.META_EXTENSION + ); + metaOut = this.state.directory.createOutput(metaFileName, this.state.context); + CodecUtil.writeIndexHeader( + metaOut, + Composite99DocValuesFormat.META_CODEC_NAME, + Composite99DocValuesFormat.VERSION_CURRENT, + this.state.segmentInfo.getId(), + this.state.segmentSuffix + ); + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } // check if there are any composite fields which are part of the segment // TODO : add integ test where there are no composite fields in a segment, test both flush and merge cases segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false; @@ -117,6 +184,26 @@ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) @Override public void close() throws IOException { delegate.close(); + boolean success = false; + try { + if (metaOut != null) { + metaOut.writeLong(-1); // write EOF marker + CodecUtil.writeFooter(metaOut); // write checksum + } + if (dataOut != null) { + CodecUtil.writeFooter(dataOut); // write checksum + } + + success = true; + } finally { + if (success) { + IOUtils.close(dataOut, metaOut, composite99DocValuesConsumer); + } else { + IOUtils.closeWhileHandlingException(dataOut, metaOut, composite99DocValuesConsumer); + } + metaOut = dataOut = null; + composite99DocValuesConsumer = null; + } } private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, FieldInfo field) throws IOException { @@ -135,9 +222,9 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, // we have all the required fields to build composite fields if (compositeFieldSet.isEmpty()) { for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { - if (mappedType.getCompositeIndexType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) { - try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService)) { - starTreesBuilder.build(fieldProducerMap); + if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { + try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer); } } } @@ -175,6 +262,7 @@ public void merge(MergeState mergeState) throws IOException { /** * Merges composite fields from multiple segments + * * @param mergeState merge state */ private void mergeCompositeFields(MergeState mergeState) throws IOException { @@ -183,6 +271,7 @@ private void mergeCompositeFields(MergeState mergeState) throws IOException { /** * Merges star tree data fields from multiple segments + * * @param mergeState merge state */ private void mergeStarTreeFields(MergeState mergeState) throws IOException { @@ -205,7 +294,7 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { CompositeIndexValues compositeIndexValues = reader.getCompositeIndexValues(fieldInfo); if (compositeIndexValues instanceof StarTreeValues) { StarTreeValues starTreeValues = (StarTreeValues) compositeIndexValues; - List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), Collections.emptyList()); + List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), new ArrayList<>()); if (starTreeField == null) { starTreeField = starTreeValues.getStarTreeField(); } @@ -223,8 +312,36 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { } } } - try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService)) { - starTreesBuilder.buildDuringMerge(starTreeSubsPerField); + try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { + starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeSubsPerField, composite99DocValuesConsumer); } } + + private static SegmentWriteState getSegmentWriteState(SegmentWriteState segmentWriteState) { + + SegmentInfo segmentInfo = new SegmentInfo( + segmentWriteState.segmentInfo.dir, + segmentWriteState.segmentInfo.getVersion(), + segmentWriteState.segmentInfo.getMinVersion(), + segmentWriteState.segmentInfo.name, + DocIdSetIterator.NO_MORE_DOCS, + segmentWriteState.segmentInfo.getUseCompoundFile(), + segmentWriteState.segmentInfo.getHasBlocks(), + segmentWriteState.segmentInfo.getCodec(), + segmentWriteState.segmentInfo.getDiagnostics(), + segmentWriteState.segmentInfo.getId(), + segmentWriteState.segmentInfo.getAttributes(), + segmentWriteState.segmentInfo.getIndexSort() + ); + + return new SegmentWriteState( + segmentWriteState.infoStream, + segmentWriteState.directory, + segmentInfo, + segmentWriteState.fieldInfos, + segmentWriteState.segUpdates, + segmentWriteState.context + ); + } + } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java b/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java deleted file mode 100644 index 8378a4063b7ca..0000000000000 --- a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeValues.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.composite.datacube.startree; - -import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.codec.composite.CompositeIndexValues; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; - -import java.util.Map; - -/** - * Concrete class that holds the star tree associated values from the segment - * - * @opensearch.experimental - */ -@ExperimentalApi -public class StarTreeValues implements CompositeIndexValues { - private final StarTreeField starTreeField; - private final StarTreeNode root; - private final Map dimensionDocValuesIteratorMap; - private final Map metricDocValuesIteratorMap; - private final Map attributes; - - public StarTreeValues( - StarTreeField starTreeField, - StarTreeNode root, - Map dimensionDocValuesIteratorMap, - Map metricDocValuesIteratorMap, - Map attributes - ) { - this.starTreeField = starTreeField; - this.root = root; - this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; - this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; - this.attributes = attributes; - } - - @Override - public CompositeIndexValues getValues() { - return this; - } - - public StarTreeField getStarTreeField() { - return starTreeField; - } - - public StarTreeNode getRoot() { - return root; - } - - public Map getDimensionDocValuesIteratorMap() { - return dimensionDocValuesIteratorMap; - } - - public Map getMetricDocValuesIteratorMap() { - return metricDocValuesIteratorMap; - } - - public Map getAttributes() { - return attributes; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java index 9402675ff39d9..81e19f6cdb1a3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java @@ -23,4 +23,9 @@ public class CompositeIndexConstants { */ public static final String SEGMENT_DOCS_COUNT = "segmentDocsCount"; + /** + * Represents the key to fetch number of total star tree documents in a segment. + */ + public static final String STAR_TREE_DOCS_COUNT = "starTreeDocsCount"; + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index d3105b4ae23c7..13d5059a241b1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -9,15 +9,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedNumericDocValuesWriterWrapper; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -26,6 +29,8 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; @@ -38,16 +43,18 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.getFieldInfo; /** * Builder for star tree. Defines the algorithm to construct star-tree @@ -76,19 +83,30 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected final InMemoryTreeNode rootNode = getNewNode(); protected final StarTreeField starTreeField; - private final SegmentWriteState state; - static String NUM_SEGMENT_DOCS = "numSegmentDocs"; + private final SegmentWriteState writeState; + + private final IndexOutput metaOut; + private final IndexOutput dataOut; /** * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * * @param starTreeField holds the configuration for the star tree - * @param state stores the segment write state + * @param writeState stores the segment write writeState * @param mapperService helps to find the original type of the field */ - protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) { + protected BaseStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState writeState, + MapperService mapperService + ) { logger.debug("Building star tree : {}", starTreeField.getName()); + this.metaOut = metaOut; + this.dataOut = dataOut; + this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); @@ -96,8 +114,8 @@ protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState sta this.numDimensions = dimensionsSplitOrder.size(); this.skipStarNodeCreationForDimensions = new HashSet<>(); - this.totalSegmentDocs = state.segmentInfo.maxDoc(); - this.state = state; + this.totalSegmentDocs = writeState.segmentInfo.maxDoc(); + this.writeState = writeState; Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); @@ -155,6 +173,227 @@ public List generateMetricAggregatorInfos(MapperService ma return metricAggregatorInfos; } + /** + * Generates the configuration required to perform aggregation for all the metrics on a field + * + * @return list of MetricAggregatorInfo + */ + public List getMetricReaders(SegmentWriteState state, Map fieldProducerMap) + throws IOException { + + List metricReaders = new ArrayList<>(); + for (Metric metric : this.starTreeField.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + SequentialDocValuesIterator metricReader; + FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); + if (metricStat.equals(MetricStat.DOC_COUNT)) { + // _doc_count is numeric field , so we convert to sortedNumericDocValues and get iterator + metricReader = getIteratorForNumericField(fieldProducerMap, metricFieldInfo, DocCountFieldMapper.NAME); + } else { + if (metricFieldInfo == null) { + metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); + } + metricReader = new SequentialDocValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ); + } + metricReaders.add(metricReader); + } + } + return metricReaders; + } + + /** + * Builds the star tree from the original segment documents + * + * @param fieldProducerMap contain s the docValues producer to get docValues associated with each field + * @param fieldNumberAcrossStarTrees maintains a counter for the number of star-tree fields + * @param starTreeDocValuesConsumer consumes the generated star-tree docValues + * @throws IOException when we are unable to build star-tree + */ + public void build( + Map fieldProducerMap, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { + long startTime = System.currentTimeMillis(); + logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); + + List metricReaders = getMetricReaders(writeState, fieldProducerMap); + List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + for (int i = 0; i < numDimensions; i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); + if (dimensionFieldInfo == null) { + dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); + } + dimensionReaders[i] = new SequentialDocValuesIterator( + fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) + ); + } + Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); + logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + build(starTreeDocumentIterator, fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); + logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + } + + /** + * Builds the star tree using sorted and aggregated star-tree Documents + * + * @param starTreeDocumentIterator contains the sorted and aggregated documents + * @param fieldNumberAcrossStarTrees maintains a counter for the number of star-tree fields + * @param starTreeDocValuesConsumer consumes the generated star-tree docValues + * @throws IOException when we are unable to build star-tree + */ + public void build( + Iterator starTreeDocumentIterator, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { + int numSegmentStarTreeDocument = totalSegmentDocs; + + appendDocumentsToStarTree(starTreeDocumentIterator); + int numStarTreeDocument = numStarTreeDocs; + logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + + if (numStarTreeDocs == 0) { + // serialize the star tree data + serializeStarTree(numStarTreeDocument, numStarTreeDocs); + return; + } + + constructStarTree(rootNode, 0, numStarTreeDocs); + int numStarTreeDocumentUnderStarNode = numStarTreeDocs - numStarTreeDocument; + logger.debug( + "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", + numStarTreeNodes, + numStarTreeDocumentUnderStarNode + ); + + createAggregatedDocs(rootNode); + int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + + // Create doc values indices in disk + createSortedDocValuesIndices(starTreeDocValuesConsumer, fieldNumberAcrossStarTrees); + + // serialize star-tree + serializeStarTree(numStarTreeDocument, numStarTreeDocs); + } + + void appendDocumentsToStarTree(Iterator starTreeDocumentIterator) throws IOException { + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + } + + private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException { + // serialize the star tree data + long dataFilePointer = dataOut.getFilePointer(); + StarTreeWriter starTreeWriter = new StarTreeWriter(); + long totalStarTreeDataLength = starTreeWriter.writeStarTree(dataOut, rootNode, numStarTreeNodes, starTreeField.getName()); + + // serialize the star tree meta + starTreeWriter.writeStarTreeMetadata( + metaOut, + starTreeField, + metricAggregatorInfos, + numStarTreeNodes, + numSegmentStarTreeDocument, + numStarTreeDocs, + dataFilePointer, + totalStarTreeDataLength + ); + } + + private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, AtomicInteger fieldNumberAcrossStarTrees) + throws IOException { + List dimensionWriters = new ArrayList<>(); + List metricWriters = new ArrayList<>(); + FieldInfo[] dimensionFieldInfoList = new FieldInfo[starTreeField.getDimensionsOrder().size()]; + FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; + for (int i = 0; i < dimensionFieldInfoList.length; i++) { + final FieldInfo fi = getFieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues( + starTreeField.getName(), + starTreeField.getDimensionsOrder().get(i).getField() + ), + DocValuesType.SORTED_NUMERIC, + fieldNumberAcrossStarTrees.getAndIncrement() + ); + dimensionFieldInfoList[i] = fi; + dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + } + for (int i = 0; i < metricAggregatorInfos.size(); i++) { + + final FieldInfo fi = getFieldInfo( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeField.getName(), + metricAggregatorInfos.get(i).getField(), + metricAggregatorInfos.get(i).getMetricStat().getTypeName() + ), + DocValuesType.SORTED_NUMERIC, + fieldNumberAcrossStarTrees.getAndIncrement() + ); + + metricFieldInfoList[i] = fi; + metricWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + } + + for (int docId = 0; docId < numStarTreeDocs; docId++) { + StarTreeDocument starTreeDocument = getStarTreeDocument(docId); + for (int i = 0; i < starTreeDocument.dimensions.length; i++) { + if (starTreeDocument.dimensions[i] != null) { + dimensionWriters.get(i).addValue(docId, starTreeDocument.dimensions[i]); + } + } + + for (int i = 0; i < starTreeDocument.metrics.length; i++) { + try { + switch (metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType()) { + case LONG: + if (starTreeDocument.metrics[i] != null) { + metricWriters.get(i).addValue(docId, (long) starTreeDocument.metrics[i]); + } + break; + case DOUBLE: + if (starTreeDocument.metrics[i] != null) { + metricWriters.get(i) + .addValue(docId, NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i])); + } + break; + default: + throw new IllegalStateException("Unknown metric doc value type"); + } + } catch (IllegalArgumentException e) { + logger.error("could not parse the value, exiting creation of star tree"); + } + } + } + + addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, starTreeField.getDimensionsOrder().size()); + addStarTreeDocValueFields(docValuesConsumer, metricWriters, metricFieldInfoList, metricAggregatorInfos.size()); + } + + private void addStarTreeDocValueFields( + DocValuesConsumer docValuesConsumer, + List docValuesWriters, + FieldInfo[] fieldInfoList, + int fieldCount + ) throws IOException { + for (int i = 0; i < fieldCount; i++) { + final int writerIndex = i; + DocValuesProducer docValuesProducer = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return docValuesWriters.get(writerIndex).getDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(fieldInfoList[i], docValuesProducer); + } + } + /** * Get star tree document from the segment for the current docId with the dimensionReaders and metricReaders */ @@ -376,6 +615,7 @@ protected StarTreeDocument reduceSegmentStarTreeDocuments( */ private static Long getLong(Object metric) { Long metricValue = null; + if (metric instanceof Long) { metricValue = (long) metric; } @@ -422,89 +662,6 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum } } - /** - * Builds the star tree from the original segment documents - * - * @param fieldProducerMap contain s the docValues producer to get docValues associated with each field - * @throws IOException when we are unable to build star-tree - */ - public void build(Map fieldProducerMap) throws IOException { - long startTime = System.currentTimeMillis(); - logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - if (totalSegmentDocs == 0) { - logger.debug("No documents found in the segment"); - return; - } - List metricReaders = getMetricReaders(state, fieldProducerMap); - List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - for (int i = 0; i < numDimensions; i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - FieldInfo dimensionFieldInfo = state.fieldInfos.fieldInfo(dimension); - if (dimensionFieldInfo == null) { - dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); - } - dimensionReaders[i] = new SequentialDocValuesIterator( - fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) - ); - } - Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); - logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); - build(starTreeDocumentIterator); - logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); - } - - private static FieldInfo getFieldInfo(String field, DocValuesType docValuesType) { - return new FieldInfo( - field, - 1, // This is filled as part of doc values creation and is not used otherwise - false, - false, - false, - IndexOptions.NONE, - docValuesType, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - } - - /** - * Generates the configuration required to perform aggregation for all the metrics on a field - * - * @return list of MetricAggregatorInfo - */ - public List getMetricReaders(SegmentWriteState state, Map fieldProducerMap) - throws IOException { - List metricReaders = new ArrayList<>(); - for (Metric metric : this.starTreeField.getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { - SequentialDocValuesIterator metricReader = null; - FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); - if (metricStat.equals(MetricStat.DOC_COUNT)) { - // _doc_count is numeric field , so we convert to sortedNumericDocValues and get iterator - metricReader = getIteratorForNumericField(fieldProducerMap, metricFieldInfo, DocCountFieldMapper.NAME); - } else { - if (metricFieldInfo == null) { - metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); - } - metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) - ); - } - metricReaders.add(metricReader); - } - } - return metricReaders; - } - /** * Converts numericDocValues to sortedNumericDocValues and returns SequentialDocValuesIterator */ @@ -524,45 +681,6 @@ private SequentialDocValuesIterator getIteratorForNumericField( return sequentialDocValuesIterator; } - /** - * Builds the star tree using Star-Tree Document - * - * @param starTreeDocumentIterator contains the sorted and aggregated documents - * @throws IOException when we are unable to build star-tree - */ - void build(Iterator starTreeDocumentIterator) throws IOException { - int numSegmentStarTreeDocument = totalSegmentDocs; - - while (starTreeDocumentIterator.hasNext()) { - appendToStarTree(starTreeDocumentIterator.next()); - } - int numStarTreeDocument = numStarTreeDocs; - logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); - - if (numStarTreeDocs == 0) { - // TODO: Uncomment when segment codec and file formats is ready - // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); - return; - } - - constructStarTree(rootNode, 0, numStarTreeDocs); - int numStarTreeDocumentUnderStarNode = numStarTreeDocs - numStarTreeDocument; - logger.debug( - "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", - numStarTreeNodes, - numStarTreeDocumentUnderStarNode - ); - - createAggregatedDocs(rootNode); - int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; - logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); - - // TODO: When StarTreeFactory Codec is ready - // Create doc values indices in disk - // Serialize and save in disk - // Write star tree metadata for off heap implementation - } - /** * Adds a document to star-tree * @@ -584,6 +702,20 @@ private InMemoryTreeNode getNewNode() { return new InMemoryTreeNode(); } + /** + * Returns a new star-tree node + * @param dimensionId dimension id of the star-tree node + * @param startDocId start doc id of the star-tree node + * @param endDocId end doc id of the star-tree node + * @param nodeType node type of the star-tree node + * @param dimensionValue dimension value of the star-tree node + * @return + */ + private InMemoryTreeNode getNewNode(int dimensionId, int startDocId, int endDocId, byte nodeType, long dimensionValue) { + numStarTreeNodes++; + return new InMemoryTreeNode(dimensionId, startDocId, endDocId, nodeType, dimensionValue); + } + /** * Implements the algorithm to construct a star-tree * @@ -594,63 +726,73 @@ private InMemoryTreeNode getNewNode() { */ private void constructStarTree(InMemoryTreeNode node, int startDocId, int endDocId) throws IOException { - int childDimensionId = node.dimensionId + 1; + int childDimensionId = node.getDimensionId() + 1; if (childDimensionId == numDimensions) { return; } // Construct all non-star children nodes - node.childDimensionId = childDimensionId; - Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); - node.children = children; + node.setChildDimensionId(childDimensionId); + constructNonStarNodes(node, startDocId, endDocId, childDimensionId); // Construct star-node if required - if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) { - children.put((long) ALL, constructStarNode(startDocId, endDocId, childDimensionId)); + if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && node.getChildren().size() > 1) { + node.addChildNode(constructStarNode(startDocId, endDocId, childDimensionId), (long) ALL); + } + + // Further split star node if needed + if (node.getChildStarNode() != null + && (node.getChildStarNode().getEndDocId() - node.getChildStarNode().getStartDocId() > maxLeafDocuments)) { + constructStarTree(node.getChildStarNode(), node.getChildStarNode().getStartDocId(), node.getChildStarNode().getEndDocId()); } // Further split on child nodes if required - for (InMemoryTreeNode child : children.values()) { - if (child.endDocId - child.startDocId > maxLeafDocuments) { - constructStarTree(child, child.startDocId, child.endDocId); + for (InMemoryTreeNode child : node.getChildren().values()) { + if (child.getEndDocId() - child.getStartDocId() > maxLeafDocuments) { + constructStarTree(child, child.getStartDocId(), child.getEndDocId()); } } + } /** * Constructs non star tree nodes * + * @param node parent node * @param startDocId start document id (inclusive) * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree - * @return root node with non-star nodes constructed + * * @throws IOException throws an exception if we are unable to construct non-star nodes */ - private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) throws IOException { - Map nodes = new HashMap<>(); + private void constructNonStarNodes(InMemoryTreeNode node, int startDocId, int endDocId, int dimensionId) throws IOException { int nodeStartDocId = startDocId; Long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); for (int i = startDocId + 1; i < endDocId; i++) { Long dimensionValue = getDimensionValue(i, dimensionId); if (Objects.equals(dimensionValue, nodeDimensionValue) == false) { - InMemoryTreeNode child = getNewNode(); - child.dimensionId = dimensionId; - child.dimensionValue = nodeDimensionValue != null ? nodeDimensionValue : ALL; - child.startDocId = nodeStartDocId; - child.endDocId = i; - nodes.put(nodeDimensionValue, child); + addChildNode(node, i, dimensionId, nodeStartDocId, nodeDimensionValue); nodeStartDocId = i; nodeDimensionValue = dimensionValue; } } - InMemoryTreeNode lastNode = getNewNode(); - lastNode.dimensionId = dimensionId; - lastNode.dimensionValue = nodeDimensionValue != null ? nodeDimensionValue : ALL; - lastNode.startDocId = nodeStartDocId; - lastNode.endDocId = endDocId; - nodes.put(nodeDimensionValue, lastNode); - return nodes; + addChildNode(node, endDocId, dimensionId, nodeStartDocId, nodeDimensionValue); + } + + private void addChildNode(InMemoryTreeNode node, int endDocId, int dimensionId, int nodeStartDocId, Long nodeDimensionValue) { + long childNodeDimensionValue; + byte childNodeType; + if (nodeDimensionValue == null) { + childNodeDimensionValue = ALL; + childNodeType = StarTreeNodeType.NULL.getValue(); + } else { + childNodeDimensionValue = nodeDimensionValue; + childNodeType = StarTreeNodeType.DEFAULT.getValue(); + } + + InMemoryTreeNode lastNode = getNewNode(dimensionId, nodeStartDocId, endDocId, childNodeType, childNodeDimensionValue); + node.addChildNode(lastNode, nodeDimensionValue); } /** @@ -663,17 +805,10 @@ private Map constructNonStarNodes(int startDocId, int en * @throws IOException throws an exception if we are unable to construct non-star nodes */ private InMemoryTreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { - InMemoryTreeNode starNode = getNewNode(); - starNode.dimensionId = dimensionId; - starNode.dimensionValue = ALL; - starNode.nodeType = StarTreeNodeType.STAR.getValue(); - starNode.startDocId = numStarTreeDocs; + int starNodeStartDocId = numStarTreeDocs; Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); - while (starTreeDocumentIterator.hasNext()) { - appendToStarTree(starTreeDocumentIterator.next()); - } - starNode.endDocId = numStarTreeDocs; - return starNode; + appendDocumentsToStarTree(starTreeDocumentIterator); + return getNewNode(dimensionId, starNodeStartDocId, numStarTreeDocs, StarTreeNodeType.STAR.getValue(), ALL); } /** @@ -685,57 +820,56 @@ private InMemoryTreeNode constructStarNode(int startDocId, int endDocId, int dim */ private StarTreeDocument createAggregatedDocs(InMemoryTreeNode node) throws IOException { StarTreeDocument aggregatedStarTreeDocument = null; - if (node.children == null) { - // For leaf node - if (node.startDocId == node.endDocId - 1) { + // For leaf node + if (!node.hasChild()) { + + if (node.getStartDocId() == node.getEndDocId() - 1) { // If it has only one document, use it as the aggregated document - aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId); - node.aggregatedDocId = node.startDocId; + aggregatedStarTreeDocument = getStarTreeDocument(node.getStartDocId()); + node.setAggregatedDocId(node.getStartDocId()); } else { // If it has multiple documents, aggregate all of them - for (int i = node.startDocId; i < node.endDocId; i++) { + for (int i = node.getStartDocId(); i < node.getEndDocId(); i++) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } - for (int i = node.dimensionId + 1; i < numDimensions; i++) { + for (int i = node.getDimensionId() + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numStarTreeDocs; + node.setAggregatedDocId(numStarTreeDocs); appendToStarTree(aggregatedStarTreeDocument); } } else { // For non-leaf node - if (node.children.containsKey((long) ALL)) { + if (node.getChildStarNode() != null) { // If it has star child, use the star child aggregated document directly - for (InMemoryTreeNode child : node.children.values()) { - if (child.nodeType == StarTreeNodeType.STAR.getValue()) { - aggregatedStarTreeDocument = createAggregatedDocs(child); - node.aggregatedDocId = child.aggregatedDocId; - } else { - createAggregatedDocs(child); - } + aggregatedStarTreeDocument = createAggregatedDocs(node.getChildStarNode()); + node.setAggregatedDocId(node.getChildStarNode().getAggregatedDocId()); + + for (InMemoryTreeNode child : node.getChildren().values()) { + createAggregatedDocs(child); } } else { // If no star child exists, aggregate all aggregated documents from non-star children - if (node.children.values().size() == 1) { - for (InMemoryTreeNode child : node.children.values()) { + if (node.getChildren().values().size() == 1) { + for (InMemoryTreeNode child : node.getChildren().values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); - node.aggregatedDocId = child.aggregatedDocId; + node.setAggregatedDocId(child.getAggregatedDocId()); } } else { - for (InMemoryTreeNode child : node.children.values()) { + for (InMemoryTreeNode child : node.getChildren().values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } - for (int i = node.dimensionId + 1; i < numDimensions; i++) { + for (int i = node.getDimensionId() + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numStarTreeDocs; + node.setAggregatedDocId(numStarTreeDocs); appendToStarTree(aggregatedStarTreeDocument); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index f63b0cb0cc77d..62fda3e56d289 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -10,14 +10,18 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeDocumentsSorter; import org.opensearch.index.mapper.MapperService; @@ -29,11 +33,15 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; /** * Off-heap implementation of the star tree builder. + * * @opensearch.experimental */ @ExperimentalApi @@ -46,12 +54,20 @@ public class OffHeapStarTreeBuilder extends BaseStarTreeBuilder { * Builds star tree based on star tree field configuration consisting of dimensions, metrics and star tree index * specific configuration. * + * @param metaOut an index output to write star-tree metadata + * @param dataOut an index output to write star-tree data * @param starTreeField holds the configuration for the star tree * @param state stores the segment write state * @param mapperService helps to find the original type of the field */ - protected OffHeapStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) throws IOException { - super(starTreeField, state, mapperService); + protected OffHeapStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { + super(metaOut, dataOut, starTreeField, state, mapperService); segmentDocumentFileManager = new SegmentDocsFileManager(state, starTreeField, metricAggregatorInfos); try { starTreeDocumentFileManager = new StarTreeDocsFileManager(state, starTreeField, metricAggregatorInfos); @@ -73,10 +89,14 @@ public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOE * @param starTreeValuesSubs contains the star tree values from multiple segments */ @Override - public void build(List starTreeValuesSubs) throws IOException { + public void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { boolean success = false; try { - build(mergeStarTrees(starTreeValuesSubs)); + build(mergeStarTrees(starTreeValuesSubs), fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); success = true; } finally { starTreeDocumentFileManager.deleteFiles(success); @@ -133,15 +153,23 @@ Iterator mergeStarTrees(List starTreeValuesSub .size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocValuesIteratorMap().get(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); } List metricReaders = new ArrayList<>(); - for (Map.Entry metricDocValuesEntry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { - metricReaders.add(new SequentialDocValuesIterator(metricDocValuesEntry.getValue())); + // get doc id set iterators for metrics + for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeValues.getStarTreeField().getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + } } int currentDocId = 0; int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) + starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) ); while (currentDocId < numSegmentDocs) { StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 8ff111d3b41d9..ec9475caf7d6d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -7,13 +7,17 @@ */ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.mapper.MapperService; @@ -22,8 +26,11 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; /** * On heap single tree builder @@ -38,12 +45,20 @@ public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { /** * Constructor for OnHeapStarTreeBuilder * + * @param metaOut an index output to write star-tree metadata + * @param dataOut an index output to write star-tree data * @param starTreeField star-tree field * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field */ - public OnHeapStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState segmentWriteState, MapperService mapperService) { - super(starTreeField, segmentWriteState, mapperService); + public OnHeapStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) throws IOException { + super(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); } @Override @@ -84,12 +99,16 @@ public Iterator sortAndAggregateSegmentDocuments( // TODO : we can save empty iterator for dimensions which are not part of segment starTreeDocuments[currentDocId] = getSegmentStarTreeDocument(currentDocId, dimensionReaders, metricReaders); } - return sortAndAggregateStarTreeDocuments(starTreeDocuments); + return sortAndAggregateStarTreeDocuments(starTreeDocuments, false); } @Override - public void build(List starTreeValuesSubs) throws IOException { - build(mergeStarTrees(starTreeValuesSubs)); + public void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { + build(mergeStarTrees(starTreeValuesSubs), fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); } /** @@ -119,17 +138,26 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocValuesIteratorMap().get(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); } List metricReaders = new ArrayList<>(); - for (Map.Entry metricDocValuesEntry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { - metricReaders.add(new SequentialDocValuesIterator(metricDocValuesEntry.getValue())); + // get doc id set iterators for metrics + for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeValues.getStarTreeField().getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + + } } int currentDocId = 0; int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) + starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) ); while (currentDocId < numSegmentDocs) { starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); @@ -140,10 +168,6 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal return starTreeDocuments.toArray(starTreeDocumentsArr); } - Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) { - return sortAndAggregateStarTreeDocuments(starTreeDocuments, false); - } - /** * Sort, aggregates and merges the star-tree documents * diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 357f48c0cc726..23415ddf29132 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -8,14 +8,16 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * A star-tree builder that builds a single star-tree. @@ -27,17 +29,29 @@ public interface StarTreeBuilder extends Closeable { /** * Builds the star tree from the original segment documents * - * @param fieldProducerMap contains the docValues producer to get docValues associated with each field + * @param fieldProducerMap contains the docValues producer to get docValues associated with each field + * @param fieldNumberAcrossStarTrees maintains the unique field number across the fields in the star tree + * @param starTreeDocValuesConsumer consumer of star-tree doc values * @throws IOException when we are unable to build star-tree */ - void build(Map fieldProducerMap) throws IOException; + void build( + Map fieldProducerMap, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException; /** - * Builds the star tree using StarTreeFactory values from multiple segments + * Builds the star tree using Star Tree values from multiple segments * * @param starTreeValuesSubs contains the star tree values from multiple segments + * @param fieldNumberAcrossStarTrees maintains the unique field number across the fields in the star tree + * @param starTreeDocValuesConsumer consumer of star-tree doc values * @throws IOException when we are unable to build star-tree */ - void build(List starTreeValuesSubs) throws IOException; + void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 3b376d7c34351..bc598c9aeab7c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -10,11 +10,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; @@ -25,6 +27,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * Builder to construct star-trees based on multiple star-tree fields. @@ -39,8 +42,9 @@ public class StarTreesBuilder implements Closeable { private final List starTreeFields; private final SegmentWriteState state; private final MapperService mapperService; + private AtomicInteger fieldNumberAcrossStarTrees; - public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mapperService) { + public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mapperService, AtomicInteger fieldNumberAcrossStarTrees) { List starTreeFields = new ArrayList<>(); for (CompositeMappedFieldType compositeMappedFieldType : mapperService.getCompositeFieldTypes()) { if (compositeMappedFieldType instanceof StarTreeMapper.StarTreeFieldType) { @@ -58,12 +62,24 @@ public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mappe this.starTreeFields = starTreeFields; this.state = segmentWriteState; this.mapperService = mapperService; + this.fieldNumberAcrossStarTrees = fieldNumberAcrossStarTrees; } /** - * Builds the star-trees. + * Builds all star-trees for given star-tree fields. + * + * @param metaOut an IndexInput for star-tree metadata + * @param dataOut an IndexInput for star-tree data + * @param fieldProducerMap fetches iterators for the fields (dimensions and metrics) + * @param starTreeDocValuesConsumer a consumer to write star-tree doc values + * @throws IOException when an error occurs while building the star-trees */ - public void build(Map fieldProducerMap) throws IOException { + public void build( + IndexOutput metaOut, + IndexOutput dataOut, + Map fieldProducerMap, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { if (starTreeFields.isEmpty()) { logger.debug("no star-tree fields found, returning from star-tree builder"); return; @@ -75,8 +91,8 @@ public void build(Map fieldProducerMap) throws IOExce // Build all star-trees for (StarTreeField starTreeField : starTreeFields) { - try (StarTreeBuilder starTreeBuilder = getStarTreeBuilder(starTreeField, state, mapperService)) { - starTreeBuilder.build(fieldProducerMap); + try (StarTreeBuilder starTreeBuilder = getStarTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService)) { + starTreeBuilder.build(fieldProducerMap, fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); } } logger.debug("Took {} ms to build {} star-trees with star-tree fields", System.currentTimeMillis() - startTime, numStarTrees); @@ -90,9 +106,17 @@ public void close() throws IOException { /** * Merges star tree fields from multiple segments * + * @param metaOut an IndexInput for star-tree metadata + * @param dataOut an IndexInput for star-tree data * @param starTreeValuesSubsPerField starTreeValuesSubs per field + * @param starTreeDocValuesConsumer a consumer to write star-tree doc values */ - public void buildDuringMerge(final Map> starTreeValuesSubsPerField) throws IOException { + public void buildDuringMerge( + IndexOutput metaOut, + IndexOutput dataOut, + final Map> starTreeValuesSubsPerField, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException { logger.debug("Starting merge of {} star-trees with star-tree fields", starTreeValuesSubsPerField.size()); long startTime = System.currentTimeMillis(); for (Map.Entry> entry : starTreeValuesSubsPerField.entrySet()) { @@ -102,8 +126,8 @@ public void buildDuringMerge(final Map> starTreeVal continue; } StarTreeField starTreeField = starTreeValuesList.get(0).getStarTreeField(); - try (StarTreeBuilder builder = getStarTreeBuilder(starTreeField, state, mapperService)) { - builder.build(starTreeValuesList); + try (StarTreeBuilder builder = getStarTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService)) { + builder.build(starTreeValuesList, fieldNumberAcrossStarTrees, starTreeDocValuesConsumer); } } logger.debug( @@ -116,13 +140,18 @@ public void buildDuringMerge(final Map> starTreeVal /** * Get star-tree builder based on build mode. */ - StarTreeBuilder getStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) - throws IOException { + StarTreeBuilder getStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { switch (starTreeField.getStarTreeConfig().getBuildMode()) { case ON_HEAP: - return new OnHeapStarTreeBuilder(starTreeField, state, mapperService); + return new OnHeapStarTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService); case OFF_HEAP: - return new OffHeapStarTreeBuilder(starTreeField, state, mapperService); + return new OffHeapStarTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService); default: throw new IllegalArgumentException( String.format( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java index 7f1839024eea7..e5890be3ccb5b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java @@ -11,7 +11,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.fileformats.data.StarTreeDataWriter; -import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetaWriter; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadataWriter; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import java.io.IOException; @@ -54,6 +54,7 @@ public long writeStarTree(IndexOutput dataOut, InMemoryTreeNode rootNode, int nu * @param metricAggregatorInfos metric aggregator infos * @param numNodes number of nodes in the star tree * @param segmentAggregatedCount segment aggregated count + * @param numStarTreeDocs the total number of star tree documents for the segment * @param dataFilePointer data file pointer * @param dataFileLength data file length * @throws IOException when star-tree data serialization fails @@ -64,15 +65,17 @@ public void writeStarTreeMetadata( List metricAggregatorInfos, Integer numNodes, Integer segmentAggregatedCount, + Integer numStarTreeDocs, long dataFilePointer, long dataFileLength ) throws IOException { - StarTreeMetaWriter.writeStarTreeMetadata( + StarTreeMetadataWriter.writeStarTreeMetadata( metaOut, starTreeField, metricAggregatorInfos, numNodes, segmentAggregatedCount, + numStarTreeDocs, dataFilePointer, dataFileLength ); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeDataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeDataWriter.java index 32feb78a4db3d..5fe70d3c075cf 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeDataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeDataWriter.java @@ -14,10 +14,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; import java.util.LinkedList; -import java.util.List; import java.util.Queue; import static org.opensearch.index.compositeindex.datacube.startree.fileformats.node.FixedLengthStarTreeNode.SERIALIZABLE_DATA_SIZE_IN_BYTES; @@ -66,21 +63,26 @@ private static void writeStarTreeNodes(IndexOutput output, InMemoryTreeNode root while (!queue.isEmpty()) { InMemoryTreeNode node = queue.remove(); - if (node.children == null || node.children.isEmpty()) { + if (!node.hasChild()) { writeStarTreeNode(output, node, ALL, ALL); } else { - // Sort all children nodes based on dimension value - List sortedChildren = new ArrayList<>(node.children.values()); - sortedChildren.sort( - Comparator.comparingInt(InMemoryTreeNode::getNodeType).thenComparingLong(InMemoryTreeNode::getDimensionValue) - ); - + int totalNumberOfChildren = 0; int firstChildId = currentNodeId + queue.size() + 1; - int lastChildId = firstChildId + sortedChildren.size() - 1; + + if (node.getChildStarNode() != null) { + totalNumberOfChildren++; + queue.add(node.getChildStarNode()); + } + + if (node.getChildren() != null) { + totalNumberOfChildren = totalNumberOfChildren + node.getChildren().values().size(); + queue.addAll(node.getChildren().values()); + } + + int lastChildId = firstChildId + totalNumberOfChildren - 1; writeStarTreeNode(output, node, firstChildId, lastChildId); - queue.addAll(sortedChildren); } currentNodeId++; @@ -97,12 +99,12 @@ private static void writeStarTreeNodes(IndexOutput output, InMemoryTreeNode root * @throws IOException if an I/O error occurs while writing the node */ private static void writeStarTreeNode(IndexOutput output, InMemoryTreeNode node, int firstChildId, int lastChildId) throws IOException { - output.writeInt(node.dimensionId); - output.writeLong(node.dimensionValue); - output.writeInt(node.startDocId); - output.writeInt(node.endDocId); - output.writeInt(node.aggregatedDocId); - output.writeByte(node.nodeType); + output.writeInt(node.getDimensionId()); + output.writeLong(node.getDimensionValue()); + output.writeInt(node.getStartDocId()); + output.writeInt(node.getEndDocId()); + output.writeInt(node.getAggregatedDocId()); + output.writeByte(node.getNodeType()); output.writeInt(firstChildId); output.writeInt(lastChildId); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 7519c85562a8c..2e20c9b9e0737 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -72,12 +72,12 @@ public class StarTreeMetadata extends CompositeIndexMetadata { /** * The total number of documents aggregated in this star-tree segment. */ - private final Integer segmentAggregatedDocCount; + private final int segmentAggregatedDocCount; /** * The maximum number of documents allowed in a leaf node. */ - private final Integer maxLeafDocs; + private final int maxLeafDocs; /** * Set of dimensions for which star node creation should be skipped. @@ -99,6 +99,11 @@ public class StarTreeMetadata extends CompositeIndexMetadata { */ private final long dataLength; + /** + * The number of star tree documents in the star tree. + */ + private final int starTreeDocCount; + /** * A star tree metadata constructor to initialize star tree metadata from the segment file (.cim) using index input. * @@ -124,6 +129,7 @@ public StarTreeMetadata( this.dimensionFields = readStarTreeDimensions(); this.metrics = readMetricEntries(); this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); + this.starTreeDocCount = readStarTreeDocCount(); this.maxLeafDocs = readMaxLeafDocs(); this.skipStarNodeCreationInDims = readSkipStarNodeCreationInDims(); this.starTreeBuildMode = readBuildMode(); @@ -146,6 +152,7 @@ public StarTreeMetadata( * @param dimensionFields list of dimension fields * @param metrics list of metric entries * @param segmentAggregatedDocCount segment aggregated doc count + * @param starTreeDocCount the total number of star tree documents for the segment * @param maxLeafDocs max leaf docs * @param skipStarNodeCreationInDims set of dimensions to skip star node creation * @param starTreeBuildMode star tree build mode @@ -161,6 +168,7 @@ public StarTreeMetadata( List dimensionFields, List metrics, Integer segmentAggregatedDocCount, + Integer starTreeDocCount, Integer maxLeafDocs, Set skipStarNodeCreationInDims, StarTreeFieldConfiguration.StarTreeBuildMode starTreeBuildMode, @@ -176,6 +184,7 @@ public StarTreeMetadata( this.dimensionFields = dimensionFields; this.metrics = metrics; this.segmentAggregatedDocCount = segmentAggregatedDocCount; + this.starTreeDocCount = starTreeDocCount; this.maxLeafDocs = maxLeafDocs; this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; this.starTreeBuildMode = starTreeBuildMode; @@ -184,7 +193,7 @@ public StarTreeMetadata( } private int readNumberOfNodes() throws IOException { - return meta.readInt(); + return meta.readVInt(); } private int readDimensionsCount() throws IOException { @@ -225,6 +234,10 @@ private int readSegmentAggregatedDocCount() throws IOException { return meta.readVInt(); } + private Integer readStarTreeDocCount() throws IOException { + return meta.readVInt(); + } + private int readMaxLeafDocs() throws IOException { return meta.readVInt(); } @@ -296,16 +309,25 @@ public List getMetrics() { * * @return the aggregated document count for the star-tree. */ - public Integer getSegmentAggregatedDocCount() { + public int getSegmentAggregatedDocCount() { return segmentAggregatedDocCount; } + /** + * Returns the total number of star tree documents in the segment + * + * @return the number of star tree documents in the segment + */ + public int getStarTreeDocCount() { + return starTreeDocCount; + } + /** * Returns the max leaf docs for the star-tree. * * @return the max leaf docs. */ - public Integer getMaxLeafDocs() { + public int getMaxLeafDocs() { return maxLeafDocs; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetaWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java similarity index 90% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetaWriter.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 2515c1efc3aed..1c04350e25047 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetaWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -29,9 +29,9 @@ * * @opensearch.experimental */ -public class StarTreeMetaWriter { +public class StarTreeMetadataWriter { - private static final Logger logger = LogManager.getLogger(StarTreeMetaWriter.class); + private static final Logger logger = LogManager.getLogger(StarTreeMetadataWriter.class); /** * Writes the star-tree metadata. @@ -51,6 +51,7 @@ public static void writeStarTreeMetadata( List metricAggregatorInfos, Integer numNodes, Integer segmentAggregatedCount, + Integer numStarTreeDocs, long dataFilePointer, long dataFileLength ) throws IOException { @@ -60,7 +61,16 @@ public static void writeStarTreeMetadata( writeMetaHeader(metaOut); // TODO: Replace the parameters with StarTreeMetadata class object - writeMeta(metaOut, metricAggregatorInfos, starTreeField, numNodes, segmentAggregatedCount, dataFilePointer, dataFileLength); + writeMeta( + metaOut, + metricAggregatorInfos, + starTreeField, + numNodes, + segmentAggregatedCount, + numStarTreeDocs, + dataFilePointer, + dataFileLength + ); logger.debug( "Star tree meta size in bytes : {} for star-tree field {}", @@ -81,6 +91,7 @@ private static void writeMetaHeader(IndexOutput metaOut) throws IOException { // version metaOut.writeVInt(VERSION_CURRENT); + } /** @@ -91,6 +102,7 @@ private static void writeMetaHeader(IndexOutput metaOut) throws IOException { * @param starTreeField the star tree field * @param numNodes number of nodes in the star tree * @param segmentAggregatedDocCount the aggregated document count for the segment + * @param numStarTreeDocs the total number of star tree documents for the segment * @param dataFilePointer the file pointer to the start of the star-tree data * @param dataFileLength the length of the star-tree data file * @throws IOException if an I/O error occurs while writing the metadata @@ -101,6 +113,7 @@ private static void writeMeta( StarTreeField starTreeField, int numNodes, Integer segmentAggregatedDocCount, + Integer numStarTreeDocs, long dataFilePointer, long dataFileLength ) throws IOException { @@ -112,7 +125,7 @@ private static void writeMeta( metaOut.writeString(CompositeMappedFieldType.CompositeFieldType.STAR_TREE.getName()); // number of nodes - metaOut.writeInt(numNodes); + metaOut.writeVInt(numNodes); // number of dimensions // TODO: Revisit the number of dimensions for timestamps (as we will split timestamp into min, hour, etc.) @@ -137,6 +150,9 @@ private static void writeMeta( // segment aggregated document count metaOut.writeVInt(segmentAggregatedDocCount); + // segment star tree document count + metaOut.writeVInt(numStarTreeDocs); + // max leaf docs metaOut.writeVInt(starTreeField.getStarTreeConfig().maxLeafDocs()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 89ac4af51e221..72c08d96d26ff 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -250,13 +250,12 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IO low++; } + int high = getInt(LAST_CHILD_ID_OFFSET); // if the current node is null node, increment the low to reduce the search space - if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, low), StarTreeNodeType.NULL) != null) { - low++; + if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, high), StarTreeNodeType.NULL) != null) { + high--; } - int high = getInt(LAST_CHILD_ID_OFFSET); - while (low <= high) { int mid = low + (high - low) / 2; FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/CompositeIndexValues.java similarity index 86% rename from server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/CompositeIndexValues.java index f8848aceab343..3c5c6b71849f2 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/CompositeIndexValues.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite; +package org.opensearch.index.compositeindex.datacube.startree.index; import org.opensearch.common.annotation.ExperimentalApi; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java new file mode 100644 index 0000000000000..8181a69f49bd4 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -0,0 +1,265 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.index; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexInput; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.CompositeIndexMetadata; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.ReadDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static org.opensearch.index.codec.composite.composite99.Composite99DocValuesReader.getSortedNumericDocValues; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; + +/** + * Concrete class that holds the star tree associated values from the segment + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeValues implements CompositeIndexValues { + + /** + * Representing the star tree field configuration. + */ + private final StarTreeField starTreeField; + + /** + * The root node of the star tree. + */ + private final StarTreeNode root; + + /** + * A map containing suppliers for DocIdSetIterators for dimensions. + */ + private final Map> dimensionDocValuesIteratorMap; + + /** + * A map containing suppliers for DocIdSetIterators for metrics. + */ + private final Map> metricDocValuesIteratorMap; + + /** + * A map containing attributes associated with the star tree values. + */ + private final Map attributes; + + /** + * Constructs a new StarTreeValues object with the provided parameters. + * Used for testing. + * + * @param starTreeField The StarTreeField object representing the star tree field configuration. + * @param root The root node of the star tree. + * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions. + * @param metricDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for metrics. + * @param attributes A map containing attributes associated with the star tree values. + */ + public StarTreeValues( + StarTreeField starTreeField, + StarTreeNode root, + Map> dimensionDocValuesIteratorMap, + Map> metricDocValuesIteratorMap, + Map attributes + ) { + this.starTreeField = starTreeField; + this.root = root; + this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; + this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; + this.attributes = attributes; + } + + /** + * Constructs a new StarTreeValues object by reading the data from the segment + * + * @param compositeIndexMetadata The CompositeIndexMetadata object containing metadata for the composite index. + * @param compositeIndexDataIn The IndexInput object for reading the composite index data. + * @param compositeDocValuesProducer The DocValuesProducer object for producing doc values. + * @param readState The SegmentReadState object representing the state of the segment being read. + * @throws IOException If an I/O error occurs while reading the data. + */ + public StarTreeValues( + CompositeIndexMetadata compositeIndexMetadata, + IndexInput compositeIndexDataIn, + DocValuesProducer compositeDocValuesProducer, + SegmentReadState readState + ) throws IOException { + + StarTreeMetadata starTreeMetadata = (StarTreeMetadata) compositeIndexMetadata; + + // build skip star node dimensions + Set skipStarNodeCreationInDims = starTreeMetadata.getSkipStarNodeCreationInDims(); + + // build dimensions + List readDimensions = new ArrayList<>(); + for (String dimension : starTreeMetadata.getDimensionFields()) { + readDimensions.add(new ReadDimension(dimension)); + } + + // star-tree field + this.starTreeField = new StarTreeField( + starTreeMetadata.getCompositeFieldName(), + readDimensions, + starTreeMetadata.getMetrics(), + new StarTreeFieldConfiguration( + starTreeMetadata.getMaxLeafDocs(), + skipStarNodeCreationInDims, + starTreeMetadata.getStarTreeBuildMode() + ) + ); + + this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata); + + // get doc id set iterators for metrics and dimensions + dimensionDocValuesIteratorMap = new LinkedHashMap<>(); + metricDocValuesIteratorMap = new LinkedHashMap<>(); + + // get doc id set iterators for dimensions + for (String dimension : starTreeMetadata.getDimensionFields()) { + dimensionDocValuesIteratorMap.put(dimension, () -> { + try { + SortedNumericDocValues dimensionSortedNumericDocValues = null; + if (readState != null) { + FieldInfo dimensionfieldInfo = readState.fieldInfos.fieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), dimension) + ); + if (dimensionfieldInfo != null) { + dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); + } + } + return getSortedNumericDocValues(dimensionSortedNumericDocValues); + } catch (IOException e) { + throw new RuntimeException("Error loading dimension DocIdSetIterator", e); + } + }); + } + + // get doc id set iterators for metrics + for (Metric metric : starTreeMetadata.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeField.getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricDocValuesIteratorMap.put(metricFullName, () -> { + try { + SortedNumericDocValues metricSortedNumericDocValues = null; + if (readState != null) { + FieldInfo metricFieldInfo = readState.fieldInfos.fieldInfo(metricFullName); + if (metricFieldInfo != null) { + metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo); + } + } + return getSortedNumericDocValues(metricSortedNumericDocValues); + } catch (IOException e) { + throw new RuntimeException("Error loading metric DocIdSetIterator", e); + } + }); + } + } + + // create star-tree attributes map + + // Create an unmodifiable view of the map + attributes = Map.of( + SEGMENT_DOCS_COUNT, + String.valueOf(starTreeMetadata.getSegmentAggregatedDocCount()), + STAR_TREE_DOCS_COUNT, + String.valueOf(starTreeMetadata.getStarTreeDocCount()) + ); + + } + + @Override + public CompositeIndexValues getValues() { + return this; + } + + /** + * Returns an object representing the star tree field configuration. + * + * @return The StarTreeField object representing the star tree field configuration. + */ + public StarTreeField getStarTreeField() { + return starTreeField; + } + + /** + * Returns the root node of the star tree. + * + * @return The root node of the star tree. + */ + public StarTreeNode getRoot() { + return root; + } + + /** + * Returns the map containing attributes associated with the star tree values. + * + * @return The map containing attributes associated with the star tree values. + */ + public Map getAttributes() { + return attributes; + } + + /** + * Returns the DocIdSetIterator for the specified dimension. + * + * @param dimension The name of the dimension. + * @return The DocIdSetIterator for the specified dimension. + */ + public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { + + if (dimensionDocValuesIteratorMap.containsKey(dimension)) { + return dimensionDocValuesIteratorMap.get(dimension).get(); + } + + return DocValues.emptySortedNumeric(); + } + + /** + * Returns the DocIdSetIterator for the specified fully qualified metric name. + * + * @param fullyQualifiedMetricName The fully qualified name of the metric. + * @return The DocIdSetIterator for the specified fully qualified metric name. + */ + public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) { + + if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { + return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + } + + return DocValues.emptySortedNumeric(); + } + +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/package-info.java similarity index 57% rename from server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/package-info.java index 67808ad51289a..06029042ab407 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/datacube/startree/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/package-info.java @@ -7,6 +7,6 @@ */ /** - * classes responsible for handling all star tree structures and operations as part of codec + * Classes responsible for handling all star tree values from the segment */ -package org.opensearch.index.codec.composite.datacube.startree; +package org.opensearch.index.compositeindex.datacube.startree.index; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/InMemoryTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/InMemoryTreeNode.java index 20f7dcf184391..c3bf4475f75c2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/InMemoryTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/InMemoryTreeNode.java @@ -7,14 +7,15 @@ */ package org.opensearch.index.compositeindex.datacube.startree.node; +import org.opensearch.common.SetOnce; import org.opensearch.common.annotation.ExperimentalApi; +import java.util.LinkedHashMap; import java.util.Map; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; /** - * /** * Represents a node in a tree data structure, specifically designed for a star-tree implementation. * A star-tree node will represent both star and non-star nodes. * @@ -23,45 +24,70 @@ @ExperimentalApi public class InMemoryTreeNode { + public InMemoryTreeNode() { + this.dimensionId = ALL; + this.startDocId = ALL; + this.endDocId = ALL; + this.nodeType = (byte) 0; + this.dimensionValue = ALL; + this.childStarNode = new SetOnce<>(); + this.children = new LinkedHashMap<>(); + } + + public InMemoryTreeNode(int dimensionId, int startDocId, int endDocId, byte nodeType, long dimensionValue) { + this.dimensionId = dimensionId; + this.startDocId = startDocId; + this.endDocId = endDocId; + this.nodeType = nodeType; + this.dimensionValue = dimensionValue; + this.childStarNode = new SetOnce<>(); + this.children = new LinkedHashMap<>(); + } + /** * The dimension id for the dimension (field) associated with this star-tree node. */ - public int dimensionId = ALL; + private final int dimensionId; /** * The starting document id (inclusive) associated with this star-tree node. */ - public int startDocId = ALL; + private final int startDocId; /** * The ending document id (exclusive) associated with this star-tree node. */ - public int endDocId = ALL; + private final int endDocId; /** * The aggregated document id associated with this star-tree node. */ - public int aggregatedDocId = ALL; + private int aggregatedDocId = ALL; /** * The child dimension identifier associated with this star-tree node. */ - public int childDimensionId = ALL; + private int childDimensionId = ALL; /** * The value of the dimension associated with this star-tree node. */ - public long dimensionValue = ALL; + private final long dimensionValue; /** * A byte indicating whether the node is star node, null node or default node (with dimension value present). */ - public byte nodeType = 0; + private byte nodeType; /** * A map containing the child nodes of this star-tree node, keyed by their dimension id. */ - public Map children; + private final Map children; + + /** + * A map containing the child star node of this star-tree node. + */ + private final SetOnce childStarNode; public long getDimensionValue() { return dimensionValue; @@ -71,4 +97,74 @@ public byte getNodeType() { return nodeType; } + public boolean hasChild() { + return !(this.children.isEmpty() && this.childStarNode.get() == null); + } + + public int getDimensionId() { + return dimensionId; + } + + public int getStartDocId() { + return startDocId; + } + + public int getEndDocId() { + return endDocId; + } + + public void setNodeType(byte nodeType) { + this.nodeType = nodeType; + } + + public void addChildNode(InMemoryTreeNode childNode, Long dimensionValue) { + if (childNode.getNodeType() == StarTreeNodeType.STAR.getValue()) { + this.childStarNode.set(childNode); + } else { + this.children.put(dimensionValue, childNode); + assert assertStarTreeChildOrder(childNode); + } + } + + public Map getChildren() { + return children; + } + + public InMemoryTreeNode getChildStarNode() { + return childStarNode.get(); + } + + public int getChildDimensionId() { + return childDimensionId; + } + + public void setChildDimensionId(int childDimensionId) { + this.childDimensionId = childDimensionId; + } + + public int getAggregatedDocId() { + return aggregatedDocId; + } + + public void setAggregatedDocId(int aggregatedDocId) { + this.aggregatedDocId = aggregatedDocId; + } + + private boolean assertStarTreeChildOrder(InMemoryTreeNode childNode) { + if (childNode.nodeType != StarTreeNodeType.NULL.getValue() && !this.children.isEmpty()) { + InMemoryTreeNode lastNode = null; + for (Map.Entry entry : this.children.entrySet()) { + lastNode = entry.getValue(); + } + assert lastNode.dimensionValue <= childNode.dimensionValue; + } else if (childNode.nodeType == StarTreeNodeType.NULL.getValue() && !this.children.isEmpty()) { + InMemoryTreeNode lastNode = null; + for (Map.Entry entry : this.children.entrySet()) { + lastNode = entry.getValue(); + } + assert lastNode.nodeType == StarTreeNodeType.NULL.getValue(); + } + return true; + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java index 4c4725e78ff15..9d7a31bda4deb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeType.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.node; /** - * Represents the different types of nodes in a StarTreeFactory data structure. + * Represents the different types of nodes in a Star Tree data structure. * *

* In order to handle different node types, we use a byte value to represent the node type. @@ -23,11 +23,11 @@ *

* The node type can be one of the following: *

    - *
  • Star Node: Represented by the value -2. A star node is a special node that represents + *
  • Star Node: Represented by the value -1. A star node is a special node that represents * all possible values for a dimension.
    • - *
  • Null Node: Represented by the value -1. A null node indicates the absence of any value + *
  • Null Node: Represented by the value 0. A null node indicates the absence of any value * for a dimension.
    • - *
  • Default Node: Represented by the value 0. A default node represents a node with an + *
  • Default Node: Represented by the value -1. A default node represents a node with an * actual dimension value.
    • *
* @@ -42,17 +42,17 @@ public enum StarTreeNodeType { * Represents a star node type. * */ - STAR("star", (byte) -2), + STAR("star", (byte) -1), /** - * Represents a null node type. + * Represents a default node type. */ - NULL("null", (byte) -1), + DEFAULT("default", (byte) 0), /** - * Represents a default node type. + * Represents a null node type. */ - DEFAULT("default", (byte) 0); + NULL("null", (byte) 1); private final String name; private final byte value; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java index 516d5b5a012ab..19d12bc6318d7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java @@ -8,5 +8,7 @@ /** * Holds classes associated with star tree node + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 400d7a1c00104..061841d3e140a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -28,6 +28,11 @@ public class SequentialDocValuesIterator { */ private final DocIdSetIterator docIdSetIterator; + /** + * The value associated with the latest document. + */ + private Long docValue; + /** * The id of the latest document. */ @@ -47,10 +52,19 @@ public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { * * @return the id of the latest document */ - int getDocId() { + public int getDocId() { return docId; } + /** + * Sets the id of the latest document. + * + * @param docId the ID of the latest document + */ + private void setDocId(int docId) { + this.docId = docId; + } + /** * Returns the DocIdSetIterator associated with this instance. * @@ -65,7 +79,7 @@ public int nextDoc(int currentDocId) throws IOException { if (docId >= currentDocId) { return docId; } - docId = this.docIdSetIterator.nextDoc(); + setDocId(this.docIdSetIterator.nextDoc()); return docId; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java index dc155df4eafca..2aae0d4ca7e29 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java @@ -74,7 +74,7 @@ public static FieldInfo[] getFieldInfoList(List fields) { int fieldNumber = 0; for (String fieldName : fields) { - fieldInfoList[fieldNumber] = getFieldInfo(fieldName, fieldNumber); + fieldInfoList[fieldNumber] = getFieldInfo(fieldName, DocValuesType.SORTED_NUMERIC, fieldNumber); fieldNumber++; } return fieldInfoList; @@ -83,10 +83,11 @@ public static FieldInfo[] getFieldInfoList(List fields) { /** * Get new field info instance for a given field name and field number * @param fieldName name of the field + * @param docValuesType doc value type of the field * @param fieldNumber number of the field * @return new field info instance */ - public static FieldInfo getFieldInfo(String fieldName, int fieldNumber) { + public static FieldInfo getFieldInfo(String fieldName, DocValuesType docValuesType, int fieldNumber) { return new FieldInfo( fieldName, fieldNumber, @@ -94,7 +95,40 @@ public static FieldInfo getFieldInfo(String fieldName, int fieldNumber) { false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, + docValuesType, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + + /** + * Get new field info instance for a given field name and field number. + * It's a dummy field info to fetch doc id set iterators based on field name. + *

+ * Actual field infos uses fieldNumberAcrossStarTrees parameter to achieve consistent + * and unique field numbers across fields and across multiple star trees + * + * @param fieldName name of the field + * @param docValuesType doc value type of the field + * @return new field info instance + */ + public static FieldInfo getFieldInfo(String fieldName, DocValuesType docValuesType) { + return new FieldInfo( + fieldName, + 0, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + docValuesType, -1, Collections.emptyMap(), 0, diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java index 34aecfc62b8b2..8a5a4a5a94ce6 100644 --- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java @@ -41,6 +41,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.ReplicationStats; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat; import org.opensearch.index.remote.RemoteSegmentStats; @@ -97,6 +98,10 @@ public class SegmentsStats implements Writeable, ToXContentFragment { Map.entry("tvd", "Term Vector Documents"), Map.entry("tvf", "Term Vector Fields"), Map.entry("liv", "Live Documents"), + Map.entry(Composite99DocValuesFormat.DATA_EXTENSION, "Composite Index"), + Map.entry(Composite99DocValuesFormat.META_EXTENSION, "Composite Index"), + Map.entry(Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, "Composite Index DocValues"), + Map.entry(Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION, "Composite Index DocValues"), Map.entry(FuzzyFilterPostingsFormat.FUZZY_FILTER_FILE_EXTENSION, "Fuzzy Filter") ); diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java similarity index 63% rename from server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java rename to server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index 54a9cc035d7a9..fa492e1adec0a 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -6,7 +6,9 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite.datacube.startree; +package org.opensearch.index.codec.composite99.datacube.startree; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -14,22 +16,34 @@ import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.MapperTestUtils; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.IndicesModule; import org.junit.After; @@ -37,9 +51,13 @@ import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.List; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; /** * Star tree doc values Lucene tests @@ -47,6 +65,19 @@ @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") public class StarTreeDocValuesFormatTests extends BaseDocValuesFormatTestCase { MapperService mapperService = null; + StarTreeFieldConfiguration.StarTreeBuildMode buildMode; + + public StarTreeDocValuesFormatTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + this.buildMode = buildMode; + } + + @ParametersFactory + public static Collection parameters() { + List parameters = new ArrayList<>(); + parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP }); + parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP }); + return parameters; + } @BeforeClass public static void createMapper() throws Exception { @@ -68,7 +99,7 @@ protected Codec getCodec() { final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); try { - createMapperService(getExpandedMapping("status", "size")); + createMapperService(getExpandedMapping()); } catch (IOException e) { throw new RuntimeException(e); } @@ -86,15 +117,18 @@ public void testStarTreeDocValues() throws IOException { doc.add(new SortedNumericDocValuesField("dv", 1)); doc.add(new SortedNumericDocValuesField("field", 1)); iw.addDocument(doc); + doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv", 1)); doc.add(new SortedNumericDocValuesField("field", 1)); iw.addDocument(doc); + doc = new Document(); iw.forceMerge(1); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); doc.add(new SortedNumericDocValuesField("field", 2)); iw.addDocument(doc); + doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); doc.add(new SortedNumericDocValuesField("field", 2)); @@ -102,17 +136,42 @@ public void testStarTreeDocValues() throws IOException { iw.forceMerge(1); iw.close(); - // TODO : validate star tree structures that got created + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 1L }, new Double[] { 2.0, 2.0, 2.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 2L, 2L }, new Double[] { 4.0, 2.0, 4.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 2.0, 2.0, 2.0 }); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, 2L }, new Double[] { 4.0, 2.0, 4.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of(StarTreeNumericType.DOUBLE, StarTreeNumericType.LONG, StarTreeNumericType.LONG), + reader.maxDoc() + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); directory.close(); } - private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException { + private XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); b.field("type", "star_tree"); b.startObject("config"); - b.field("max_leaf_docs", 100); + b.field("max_leaf_docs", 1); b.startArray("ordered_dimensions"); b.startObject(); b.field("name", "sndv"); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java new file mode 100644 index 0000000000000..53c9e05ccb490 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -0,0 +1,297 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.apache.lucene.store.IndexInput; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.mapper.CompositeMappedFieldType; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.COMPOSITE_FIELD_MARKER; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class StarTreeTestUtils { + + public static StarTreeDocument[] getSegmentsStarTreeDocuments( + List starTreeValuesSubs, + List starTreeNumericTypes, + int numDocs + ) throws IOException { + List starTreeDocuments = new ArrayList<>(); + for (StarTreeValues starTreeValues : starTreeValuesSubs) { + List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + + for (int i = 0; i < dimensionsSplitOrder.size(); i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + } + + List metricReaders = new ArrayList<>(); + // get doc id set iterators for metrics + for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeValues.getStarTreeField().getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + + } + } + int currentDocId = 0; + while (currentDocId < numDocs) { + starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, starTreeNumericTypes)); + currentDocId++; + } + } + StarTreeDocument[] starTreeDocumentsArr = new StarTreeDocument[starTreeDocuments.size()]; + return starTreeDocuments.toArray(starTreeDocumentsArr); + } + + public static StarTreeDocument getStarTreeDocument( + int currentDocId, + SequentialDocValuesIterator[] dimensionReaders, + List metricReaders, + List starTreeNumericTypes + ) throws IOException { + Long[] dims = new Long[dimensionReaders.length]; + int i = 0; + for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { + dimensionDocValueIterator.nextDoc(currentDocId); + Long val = dimensionDocValueIterator.value(currentDocId); + dims[i] = val; + i++; + } + i = 0; + Object[] metrics = new Object[metricReaders.size()]; + for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { + metricDocValuesIterator.nextDoc(currentDocId); + metrics[i] = toStarTreeNumericTypeValue(metricDocValuesIterator.value(currentDocId), starTreeNumericTypes.get(i)); + i++; + } + return new StarTreeDocument(dims, metrics); + } + + public static Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType starTreeNumericType) { + try { + return starTreeNumericType.getDoubleValue(value); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); + } + } + + public static void assertStarTreeDocuments(StarTreeDocument[] starTreeDocuments, StarTreeDocument[] expectedStarTreeDocuments) { + + assertNotNull(starTreeDocuments); + assertEquals(starTreeDocuments.length, expectedStarTreeDocuments.length); + + for (int i = 0; i < starTreeDocuments.length; i++) { + + StarTreeDocument resultStarTreeDocument = starTreeDocuments[i]; + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocuments[i]; + + assertNotNull(resultStarTreeDocument.dimensions); + assertNotNull(resultStarTreeDocument.metrics); + + assertEquals(resultStarTreeDocument.dimensions.length, expectedStarTreeDocument.dimensions.length); + assertEquals(resultStarTreeDocument.metrics.length, expectedStarTreeDocument.metrics.length); + + for (int di = 0; di < resultStarTreeDocument.dimensions.length; di++) { + assertEquals(resultStarTreeDocument.dimensions[di], expectedStarTreeDocument.dimensions[di]); + } + + for (int mi = 0; mi < resultStarTreeDocument.metrics.length; mi++) { + if (expectedStarTreeDocument.metrics[mi] instanceof Long) { + assertEquals(resultStarTreeDocument.metrics[mi], ((Long) expectedStarTreeDocument.metrics[mi]).doubleValue()); + } else { + assertEquals(resultStarTreeDocument.metrics[mi], expectedStarTreeDocument.metrics[mi]); + } + } + } + } + + public static void validateFileFormats( + IndexInput dataIn, + IndexInput metaIn, + InMemoryTreeNode rootNode, + StarTreeMetadata expectedStarTreeMetadata + ) throws IOException { + long magicMarker = metaIn.readLong(); + assertEquals(COMPOSITE_FIELD_MARKER, magicMarker); + int version = metaIn.readVInt(); + assertEquals(VERSION_CURRENT, version); + + String compositeFieldName = metaIn.readString(); + assertEquals(expectedStarTreeMetadata.getStarTreeFieldName(), compositeFieldName); + CompositeMappedFieldType.CompositeFieldType compositeFieldType = CompositeMappedFieldType.CompositeFieldType.fromName( + metaIn.readString() + ); + assertEquals(STAR_TREE, compositeFieldType); + StarTreeMetadata resultStarTreeMetadata = new StarTreeMetadata(metaIn, compositeFieldName, compositeFieldType, version); + assertStarTreeMetadata(expectedStarTreeMetadata, resultStarTreeMetadata); + + IndexInput starTreeIndexInput = dataIn.slice( + "star-tree data slice for respective star-tree fields", + resultStarTreeMetadata.getDataStartFilePointer(), + resultStarTreeMetadata.getDataLength() + ); + + StarTreeNode starTreeNode = StarTreeFactory.createStarTree(starTreeIndexInput, resultStarTreeMetadata); + Queue expectedTreeNodeQueue = new ArrayDeque<>(); + Queue resultTreeNodeQueue = new ArrayDeque<>(); + + expectedTreeNodeQueue.add(starTreeNode); + resultTreeNodeQueue.add(rootNode); + + while ((starTreeNode = expectedTreeNodeQueue.poll()) != null && (rootNode = resultTreeNodeQueue.poll()) != null) { + + // verify the star node + assertStarTreeNode(starTreeNode, rootNode); + + Iterator expectedChildrenIterator = starTreeNode.getChildrenIterator(); + + List sortedChildren = new ArrayList<>(); + if (rootNode.getChildren() != null) { + sortedChildren = new ArrayList<>(rootNode.getChildren().values()); + } + + if (starTreeNode.getChildDimensionId() != -1) { + assertFalse(sortedChildren.isEmpty()); + int childCount = 0; + boolean childStarNodeAsserted = false; + while (expectedChildrenIterator.hasNext()) { + StarTreeNode child = expectedChildrenIterator.next(); + InMemoryTreeNode resultChildNode = null; + if (!childStarNodeAsserted && rootNode.getChildStarNode() != null) { + // check if star tree node exists + resultChildNode = rootNode.getChildStarNode(); + assertNotNull(child); + assertNotNull(starTreeNode.getChildStarNode()); + assertStarTreeNode(child, resultChildNode); + childStarNodeAsserted = true; + } else { + resultChildNode = sortedChildren.get(childCount); + assertNotNull(child); + assertNotNull(resultChildNode); + if (child.getStarTreeNodeType() != StarTreeNodeType.NULL.getValue()) { + assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); + } else { + StarTreeNode finalStarTreeNode = starTreeNode; + assertThrows( + AssertionError.class, + () -> finalStarTreeNode.getChildForDimensionValue(child.getDimensionValue()) + ); + } + assertStarTreeNode(child, resultChildNode); + assertNotEquals(child.getStarTreeNodeType(), StarTreeNodeType.STAR.getValue()); + childCount++; + } + + expectedTreeNodeQueue.add(child); + resultTreeNodeQueue.add(resultChildNode); + } + + assertEquals(childCount, rootNode.getChildren().size()); + } else { + assertTrue(rootNode.getChildren().isEmpty()); + } + } + + assertTrue(expectedTreeNodeQueue.isEmpty()); + assertTrue(resultTreeNodeQueue.isEmpty()); + + } + + public static void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { + assertEquals(starTreeNode.getDimensionId(), treeNode.getDimensionId()); + assertEquals(starTreeNode.getDimensionValue(), treeNode.getDimensionValue()); + assertEquals(starTreeNode.getStartDocId(), treeNode.getStartDocId()); + assertEquals(starTreeNode.getEndDocId(), treeNode.getEndDocId()); + assertEquals(starTreeNode.getChildDimensionId(), treeNode.getChildDimensionId()); + assertEquals(starTreeNode.getAggregatedDocId(), treeNode.getAggregatedDocId()); + + if (starTreeNode.getChildDimensionId() != -1) { + assertFalse(starTreeNode.isLeaf()); + if (treeNode.getChildren() != null) { + assertEquals( + starTreeNode.getNumChildren(), + treeNode.getChildren().values().size() + (treeNode.getChildStarNode() != null ? 1 : 0) + ); + } + } else { + assertTrue(starTreeNode.isLeaf()); + } + + } + + public static void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetadata, StarTreeMetadata resultStarTreeMetadata) { + + assertEquals(expectedStarTreeMetadata.getCompositeFieldName(), resultStarTreeMetadata.getCompositeFieldName()); + assertEquals(expectedStarTreeMetadata.getCompositeFieldType(), resultStarTreeMetadata.getCompositeFieldType()); + assertEquals(expectedStarTreeMetadata.getDimensionFields().size(), resultStarTreeMetadata.getDimensionFields().size()); + for (int i = 0; i < expectedStarTreeMetadata.getDimensionFields().size(); i++) { + assertEquals(expectedStarTreeMetadata.getDimensionFields().get(i), resultStarTreeMetadata.getDimensionFields().get(i)); + } + assertEquals(expectedStarTreeMetadata.getMetrics().size(), resultStarTreeMetadata.getMetrics().size()); + + for (int i = 0; i < expectedStarTreeMetadata.getMetrics().size(); i++) { + + Metric expectedMetric = expectedStarTreeMetadata.getMetrics().get(i); + Metric resultMetric = resultStarTreeMetadata.getMetrics().get(i); + assertEquals(expectedMetric.getField(), resultMetric.getField()); + + for (int j = 0; j < expectedMetric.getMetrics().size(); j++) { + assertEquals(expectedMetric.getMetrics().get(j), resultMetric.getMetrics().get(j)); + } + + } + + assertEquals(expectedStarTreeMetadata.getSegmentAggregatedDocCount(), resultStarTreeMetadata.getSegmentAggregatedDocCount()); + assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), resultStarTreeMetadata.getStarTreeDocCount()); + assertEquals(expectedStarTreeMetadata.getMaxLeafDocs(), resultStarTreeMetadata.getMaxLeafDocs()); + assertEquals( + expectedStarTreeMetadata.getSkipStarNodeCreationInDims().size(), + resultStarTreeMetadata.getSkipStarNodeCreationInDims().size() + ); + for (String skipDimension : expectedStarTreeMetadata.getSkipStarNodeCreationInDims()) { + assertTrue(resultStarTreeMetadata.getSkipStarNodeCreationInDims().contains(skipDimension)); + } + assertEquals(expectedStarTreeMetadata.getStarTreeBuildMode(), resultStarTreeMetadata.getStarTreeBuildMode()); + assertEquals(expectedStarTreeMetadata.getDataStartFilePointer(), resultStarTreeMetadata.getDataStartFilePointer()); + assertEquals(expectedStarTreeMetadata.getDataLength(), resultStarTreeMetadata.getDataLength()); + assertEquals(0, (resultStarTreeMetadata.getDataLength()) % 33); + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 33088e8ccbcb3..79d1b0e4b9785 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValues; @@ -15,8 +16,10 @@ import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; @@ -24,11 +27,18 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; +import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; +import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.CompositeIndexConstants; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -36,6 +46,10 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; @@ -65,8 +79,14 @@ import java.util.Queue; import java.util.Set; import java.util.UUID; - -import static org.opensearch.index.compositeindex.datacube.startree.builder.BaseStarTreeBuilder.NUM_SEGMENT_DOCS; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -80,7 +100,12 @@ public abstract class AbstractStarTreeBuilderTests extends OpenSearchTestCase { protected StarTreeField compositeField; protected Map fieldProducerMap; protected SegmentWriteState writeState; - private BaseStarTreeBuilder builder; + protected BaseStarTreeBuilder builder; + protected IndexOutput dataOut; + protected IndexOutput metaOut; + protected DocValuesConsumer docValuesConsumer; + protected String dataFileName; + protected String metaFileName; @Before public void setup() throws IOException { @@ -107,7 +132,7 @@ public void setup() throws IOException { "test", dimensionsOrder, metrics, - new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) ); directory = newFSDirectory(createTempDir()); @@ -135,7 +160,21 @@ public void setup() throws IOException { ); fieldProducerMap.put(fields.get(i), docValuesProducer); } - writeState = getWriteState(5); + writeState = getWriteState(5, UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8)); + + dataFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.DATA_EXTENSION + ); + dataOut = writeState.directory.createOutput(dataFileName, writeState.context); + + metaFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.META_EXTENSION + ); + metaOut = writeState.directory.createOutput(metaFileName, writeState.context); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -159,9 +198,91 @@ public void setup() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); + docValuesConsumer = mock(DocValuesConsumer.class); + } + + private SegmentReadState getReadState(int numDocs, List dimensionFields, List metrics) { + + int numMetrics = 0; + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getMetrics()) { + numMetrics++; + } + } + + FieldInfo[] fields = new FieldInfo[dimensionFields.size() + numMetrics]; + + int i = 0; + for (String dimension : dimensionFields) { + fields[i] = new FieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeField.getName(), dimension), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + i++; + } + + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getMetrics()) { + fields[i] = new FieldInfo( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + metric.getField(), + metricStat.getTypeName() + ), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + i++; + } + } + + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + numDocs, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + writeState.segmentInfo.getId(), + new HashMap<>(), + null + ); + return new SegmentReadState(segmentInfo.dir, segmentInfo, new FieldInfos(fields), writeState.context); } - private SegmentWriteState getWriteState(int numDocs) { + private SegmentWriteState getWriteState(int numDocs, byte[] id) { FieldInfos fieldInfos = new FieldInfos(fieldsInfo); SegmentInfo segmentInfo = new SegmentInfo( directory, @@ -173,7 +294,7 @@ private SegmentWriteState getWriteState(int numDocs) { false, new Lucene99Codec(), new HashMap<>(), - UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + id, new HashMap<>(), null ); @@ -181,6 +302,8 @@ private SegmentWriteState getWriteState(int numDocs) { } public abstract BaseStarTreeBuilder getStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, StarTreeField starTreeField, SegmentWriteState segmentWriteState, MapperService mapperService @@ -233,7 +356,7 @@ public void test_sortAndAggregateStarTreeDocuments() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -329,7 +452,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOExcepti } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -400,7 +523,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullMetricField() throws IOEx } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -422,30 +545,35 @@ public void test_sortAndAggregateStarTreeDocuments_nullMetricField() throws IOEx } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/14813") public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionField() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; // Setting second metric iterator as empty sorted numeric , indicating a metric field is null starTreeDocuments[0] = new StarTreeDocument( new Long[] { 2L, null, 3L, 4L }, - new Object[] { 12.0, null, randomDouble(), 8.0, 20.0 } + new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, null } ); starTreeDocuments[1] = new StarTreeDocument( new Long[] { null, 4L, 2L, 1L }, - new Object[] { 10.0, null, randomDouble(), 12.0, 10.0 } + new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, null } ); starTreeDocuments[2] = new StarTreeDocument( new Long[] { null, 4L, 2L, 1L }, - new Object[] { 14.0, null, randomDouble(), 6.0, 24.0 } + new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, null, 3L, 4L }, + new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { -1L, 4L, 2L, 1L }, + new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, null } ); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, null, 3L, 4L }, new Object[] { 9.0, null, randomDouble(), 9.0, 12.0 }); - starTreeDocuments[4] = new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 11.0, null, randomDouble(), 8.0, 13.0 }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, null, 3L, 4L }, new Object[] { 21.0, 0.0, 2L }), - new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 24.0, 0.0, 2L }), - new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 11.0, 0.0, 1L }) + new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 11.0, 0.0, 1L, 8.0, 13.0, 1L }), + new StarTreeDocument(new Long[] { 2L, null, 3L, 4L }, new Object[] { 21.0, 0.0, 2L, 8.0, 20.0, 11L }), + new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 24.0, 0.0, 2L, 6.0, 24.0, 2L }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -458,20 +586,22 @@ public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionFie long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = starTreeDocuments[i].metrics[5] != null ? (long) starTreeDocuments[i].metrics[5] : null; segmentStarTreeDocuments[i] = new StarTreeDocument( starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5 } + new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } ); } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - for (StarTreeDocument resultStarTreeDocument : builder.getStarTreeDocuments()) { + while (segmentStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); @@ -484,7 +614,10 @@ public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionFie assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); } - builder.build(segmentStarTreeDocumentIterator); + + assertFalse(expectedStarTreeDocumentIterator.hasNext()); + + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -542,7 +675,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -562,7 +695,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); } - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -619,7 +752,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -639,7 +772,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); } - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -691,7 +824,7 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx } SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -760,7 +893,7 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -836,7 +969,7 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators @@ -860,7 +993,7 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() } assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); } @@ -968,19 +1101,53 @@ public void test_build_halfFloatMetrics() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(7, resultStarTreeDocuments.size()); - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - builder.build(expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); } public void test_build_floatMetrics() throws IOException { @@ -1048,20 +1215,57 @@ public void test_build_floatMetrics() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - builder.build(segmentStarTreeDocumentIterator); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(7, resultStarTreeDocuments.size()); - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); } + abstract StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode(); + public void test_build_longMetrics() throws IOException { mapperService = mock(MapperService.class); @@ -1111,22 +1315,57 @@ public void test_build_longMetrics() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(7, resultStarTreeDocuments.size()); - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); } - private static Iterator getExpectedStarTreeDocumentIterator() { - List expectedStarTreeDocuments = List.of( + private static List getExpectedStarTreeDocumentIterator() { + return List.of( new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), @@ -1135,7 +1374,220 @@ private static Iterator getExpectedStarTreeDocumentIterator() new StarTreeDocument(new Long[] { null, 4L, null, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), new StarTreeDocument(new Long[] { null, 4L, null, null }, new Object[] { 56.0, 48.0, 5L, 6.0, 24.0, 5L }) ); - return expectedStarTreeDocuments.iterator(); + } + + public void test_build_multipleStarTrees() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble(), 8.0, 13.0 }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5 } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.VALUE_COUNT)), + new Metric("field9", List.of(MetricStat.MIN)), + new Metric("field10", List.of(MetricStat.MAX)) + ); + + compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) + ); + + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + builder.close(); + + // building another tree in the same file + fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); + + dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); + metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM))); + + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 7, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + fieldProducerMap = new HashMap<>(); + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + fieldProducerMap.put(fields.get(i), docValuesProducer); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + InMemoryTreeNode rootNode1 = builder.getRootNode(); + + int noOfStarTreeDocuments2 = 7; + StarTreeDocument[] starTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; + starTreeDocuments2[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Double[] { 400.0 }); + starTreeDocuments2[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Double[] { 200.0 }); + starTreeDocuments2[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Double[] { 300.0 }); + starTreeDocuments2[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Double[] { 100.0 }); + starTreeDocuments2[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Double[] { 600.0 }); + starTreeDocuments2[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Double[] { 200.0 }); + starTreeDocuments2[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Double[] { 400.0 }); + + StarTreeDocument[] segmentStarTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; + for (int i = 0; i < noOfStarTreeDocuments2; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments2[i].metrics[0]); + segmentStarTreeDocuments2[i] = new StarTreeDocument(starTreeDocuments2[i].dimensions, new Long[] { metric1 }); + } + + SequentialDocValuesIterator[] dimsIterators2 = getDimensionIterators(segmentStarTreeDocuments2); + List metricsIterators2 = getMetricIterators(segmentStarTreeDocuments2); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator2 = builder.sortAndAggregateSegmentDocuments( + dimsIterators2, + metricsIterators2 + ); + builder.build(segmentStarTreeDocumentIterator2, new AtomicInteger(), mock(DocValuesConsumer.class)); + InMemoryTreeNode rootNode2 = builder.getRootNode(); + + metaOut.close(); + dataOut.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3", "field5", "field8"), + List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.VALUE_COUNT)), + new Metric("field9", List.of(MetricStat.MIN)), + new Metric("field10", List.of(MetricStat.MAX)) + ), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("fieldC", "fieldB", "fieldL"), + List.of(new Metric("fieldI", List.of(MetricStat.SUM))), + 7, + 27, + 1, + Set.of(), + getBuildMode(), + 330, + 1287 + ); + + List totalDimensionFields = new ArrayList<>(); + totalDimensionFields.addAll(starTreeMetadata.getDimensionFields()); + totalDimensionFields.addAll(starTreeMetadata2.getDimensionFields()); + + List metrics = new ArrayList<>(); + metrics.addAll(starTreeMetadata.getMetrics()); + metrics.addAll(starTreeMetadata2.getMetrics()); + + SegmentReadState readState = getReadState(3, totalDimensionFields, metrics); + + IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); + IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + + validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata); + validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2); + + dataIn.close(); + metaIn.close(); + } public void test_build() throws IOException { @@ -1180,18 +1632,53 @@ public void test_build() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - builder.build(segmentStarTreeDocumentIterator); + docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(7, resultStarTreeDocuments.size()); - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); } private void assertStarTreeDocuments( @@ -1224,12 +1711,7 @@ public void test_build_starTreeDataset() throws IOException { DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - compositeField = new StarTreeField( - "test", - dimensionsOrder, - metrics, - new StarTreeFieldConfiguration(1, Set.of(), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) - ); + compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, @@ -1271,7 +1753,13 @@ public void test_build_starTreeDataset() throws IOException { } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); - + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); when(mapperService.documentMapper()).thenReturn(documentMapper); @@ -1305,18 +1793,18 @@ public void test_build_starTreeDataset() throws IOException { SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(compositeField, writeState, mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - builder.build(segmentStarTreeDocumentIterator); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List resultStarTreeDocuments = builder.getStarTreeDocuments(); - Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments(); + Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments().iterator(); Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); Map> dimValueToDocIdMap = new HashMap<>(); - builder.rootNode.nodeType = StarTreeNodeType.STAR.getValue(); + builder.rootNode.setNodeType(StarTreeNodeType.STAR.getValue()); traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); Map> expectedDimToValueMap = getExpectedDimToValueMap(); @@ -1342,7 +1830,88 @@ public void test_build_starTreeDataset() throws IOException { assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); } + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 7, + 27, + 1, + Set.of(), + getBuildMode(), + 0, + 1287 + ); + validateStarTreeFileFormats(builder.getRootNode(), 27, starTreeMetadata, expectedStarTreeDocuments()); + } + + private List getStarTreeDimensionNames(List dimensionsOrder) { + + List dimensionNames = new ArrayList<>(); + for (Dimension dimension : dimensionsOrder) { + dimensionNames.add(dimension.getField()); + } + return dimensionNames; + + } + + private void validateStarTreeFileFormats( + InMemoryTreeNode rootNode, + int numDocs, + StarTreeMetadata expectedStarTreeMetadata, + List expectedStarTreeDocuments + ) throws IOException { + + assertNotNull(rootNode.getChildren()); + assertFalse(rootNode.getChildren().isEmpty()); + SegmentReadState readState = getReadState( + numDocs, + expectedStarTreeMetadata.getDimensionFields(), + expectedStarTreeMetadata.getMetrics() + ); + + DocValuesProducer compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( + Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + readState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + + IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); + IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + + StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); + List starTreeNumericTypes = new ArrayList<>(); + builder.metricAggregatorInfos.forEach( + metricAggregatorInfo -> starTreeNumericTypes.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) + ); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + starTreeNumericTypes, + readState.segmentInfo.maxDoc() + ); + + StarTreeDocument[] expectedStarTreeDocumentsArray = expectedStarTreeDocuments.toArray(new StarTreeDocument[0]); + StarTreeTestUtils.assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocumentsArray); + + validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata); + + dataIn.close(); + metaIn.close(); + compositeDocValuesProducer.close(); } private static Map> getExpectedDimToValueMap() { @@ -1367,8 +1936,8 @@ private static Map> getExpectedDimToValueMap() { return expectedDimToValueMap; } - private Iterator expectedStarTreeDocuments() { - List expectedStarTreeDocuments = List.of( + private List expectedStarTreeDocuments() { + return List.of( new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, 1L }), new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, 1L }), new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, 1L }), @@ -1398,7 +1967,6 @@ private Iterator expectedStarTreeDocuments() { new StarTreeDocument(new Long[] { 3L, 12L, null }, new Object[] { 600.0, 2L }) ); - return expectedStarTreeDocuments.iterator(); } public void testFlushFlow() throws IOException { @@ -1417,13 +1985,14 @@ public void testFlushFlow() throws IOException { ); List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - StarTreeField sf = getStarTreeFieldWithMultipleMetrics(); + compositeField = getStarTreeFieldWithMultipleMetrics(); SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); - builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimDvs, @@ -1438,19 +2007,62 @@ public void testFlushFlow() throws IOException { [5, 5] | [50.0, 1] [null, 2] | [20.0, 1] */ - int count = 0; - while (starTreeDocumentIterator.hasNext()) { - count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals( - starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 1 * 10.0 : 20.0, - starTreeDocument.metrics[0] - ); - assertEquals(1L, starTreeDocument.metrics[1]); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + int count = 0; + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + count++; + if (starTreeDocument.dimensions[1] != null) { + assertEquals( + starTreeDocument.dimensions[0] == null + ? starTreeDocument.dimensions[1] * 1 * 10.0 + : starTreeDocument.dimensions[0] * 10, + starTreeDocument.metrics[0] + ); + assertEquals(1L, starTreeDocument.metrics[1]); + } else { + assertEquals(150D, starTreeDocument.metrics[0]); + assertEquals(6L, starTreeDocument.metrics[1]); + } } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + assertEquals(13, count); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } public void testFlushFlowDimsReverse() throws IOException { @@ -1469,13 +2081,21 @@ public void testFlushFlowDimsReverse() throws IOException { ); List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - StarTreeField sf = getStarTreeFieldWithMultipleMetrics(); + compositeField = getStarTreeFieldWithMultipleMetrics(); SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); - builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimDvs, @@ -1490,21 +2110,49 @@ public void testFlushFlowDimsReverse() throws IOException { [5, 5] | [50.0, 1] [null, 0] | [0.0, 1] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { - count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] != null) { - assertEquals(count, (long) starTreeDocument.dimensions[0]); - } else { - assertEquals(6, count); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + if (count <= 6) { + count++; + if (starTreeDocument.dimensions[0] != null) { + assertEquals(count, (long) starTreeDocument.dimensions[0]); + } + assertEquals(starTreeDocument.dimensions[1] * 10.0, starTreeDocument.metrics[0]); + assertEquals(1L, starTreeDocument.metrics[1]); } - assertEquals(starTreeDocument.dimensions[1] * 10.0, starTreeDocument.metrics[0]); - assertEquals(1L, starTreeDocument.metrics[1]); } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testFlushFlowBuild() throws IOException { @@ -1534,23 +2182,28 @@ public void testFlushFlowBuild() throws IOException { Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); List dims = List.of(d1, d2); List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1, - new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP - ); - StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - builder = getStarTreeBuilder(sf, getWriteState(100), mapperService); + writeState = getWriteState(100, writeState.segmentInfo.getId()); + SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + consumerWriteState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); DocValuesProducer d1vp = getDocValuesProducer(d1sndv); DocValuesProducer d2vp = getDocValuesProducer(d2sndv); DocValuesProducer m1vp = getDocValuesProducer(m1sndv); Map fieldProducerMap = Map.of("field1", d1vp, "field3", d2vp, "field2", m1vp); - builder.build(fieldProducerMap); + builder.build(fieldProducerMap, new AtomicInteger(), docValuesConsumer); /** * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] [0, 0] | [0.0] @@ -1572,6 +2225,34 @@ public void testFlushFlowBuild() throws IOException { ); } validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 100, + builder.numStarTreeDocs, + 1, + Set.of(), + getBuildMode(), + 0, + 6699 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } private static DocValuesProducer getDocValuesProducer(SortedNumericDocValues sndv) { @@ -1583,20 +2264,15 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOExcepti }; } - private static StarTreeField getStarTreeFieldWithMultipleMetrics() { + private StarTreeField getStarTreeFieldWithMultipleMetrics() { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); List dims = List.of(d1, d2); List metrics = List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1000, - new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP - ); - StarTreeField sf = new StarTreeField("sf", dims, metrics, c); - return sf; + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); } public void testMergeFlow_randomNumberTypes() throws Exception { @@ -1631,8 +2307,6 @@ public void testMergeFlow_randomNumberTypes() throws Exception { ); when(documentMapper.mappers()).thenReturn(fieldMappers); testMergeFlowWithSum(); - builder.close(); - testMergeFlowWithCount(); } public void testMergeFlowWithSum() throws IOException { @@ -1653,12 +2327,12 @@ public void testMergeFlowWithSum() throws IOException { ); List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - StarTreeField sf = getStarTreeField(MetricStat.SUM); + compositeField = getStarTreeField(MetricStat.SUM); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -1666,10 +2340,18 @@ public void testMergeFlowWithSum() throws IOException { getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); - builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] @@ -1682,18 +2364,49 @@ public void testMergeFlowWithSum() throws IOException { * ------------------ We only take non star docs * [6,-1] | [120.0] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals( - starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, - starTreeDocument.metrics[0] - ); + if (count <= 6) { + assertEquals( + starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, + starTreeDocument.metrics[0] + ); + } } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithCount() throws IOException { @@ -1705,12 +2418,12 @@ public void testMergeFlowWithCount() throws IOException { List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -1718,10 +2431,18 @@ public void testMergeFlowWithCount() throws IOException { getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); - builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -1734,15 +2455,46 @@ public void testMergeFlowWithCount() throws IOException { --------------- [6,-1] | [12] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals(starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 : 4, starTreeDocument.metrics[0]); + if (count <= 6) { + assertEquals(starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 : 4, starTreeDocument.metrics[0]); + } } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } @@ -1756,14 +2508,26 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d1sndv = dimList; SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; - Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv); - Map metricDocIdSetIterators = Map.of("field2", m1sndv); + Map> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv); + + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + for (Metric metric : sf.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricDocIdSetIterators.put(metricFullName, () -> m1sndv); + } + } + StarTreeValues starTreeValues = new StarTreeValues( sf, null, dimDocIdSetIterators, metricDocIdSetIterators, - Map.of("numSegmentDocs", number) + Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number) ); return starTreeValues; } @@ -1785,12 +2549,12 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -1798,10 +2562,18 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -1815,19 +2587,49 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { [null, 2] | [2] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(9, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { - assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + if (count <= 9) { + if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { + assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } } } - assertEquals(9, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 9, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowNumSegmentsDocs() throws IOException { @@ -1863,7 +2665,7 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { sf, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + builder = getStarTreeBuilder(metaOut, dataOut, sf, getWriteState(4, writeState.segmentInfo.getId()), mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -1907,12 +2709,12 @@ public void testMergeFlowWithMissingDocs() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -1920,10 +2722,18 @@ public void testMergeFlowWithMissingDocs() throws IOException { getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -1938,18 +2748,49 @@ public void testMergeFlowWithMissingDocs() throws IOException { [null, 5] | [5] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] == null) { - assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + if (count <= 10) { + if (starTreeDocument.dimensions[0] == null) { + assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(10, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 10, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 363 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithMissingDocsWithZero() throws IOException { @@ -1969,12 +2810,12 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "7" ); @@ -1982,10 +2823,19 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + consumerWriteState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -1996,23 +2846,54 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { [null, 7] | [7] [null, null] | [12] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { - assertEquals(12L, (long) starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == null) { - assertEquals(7L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == 0) { - assertEquals(9L, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + if (count <= 6) { + if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { + assertEquals(12L, (long) starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == null) { + assertEquals(7L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == 0) { + assertEquals(9L, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } } } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 231 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException { @@ -2032,12 +2913,12 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "9" ); @@ -2045,10 +2926,18 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -2060,26 +2949,57 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException [null, 7] | [7] [null, null] | [19] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(7, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { - assertEquals(19L, (long) starTreeDocument.metrics[0]); - assertEquals(7, count); - } else if (starTreeDocument.dimensions[0] == null) { - assertEquals(7L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[1] == null) { - assertEquals(8L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == 0) { - assertEquals(9L, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + if (count <= 7) { + if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { + assertEquals(19L, (long) starTreeDocument.metrics[0]); + assertEquals(7, count); + } else if (starTreeDocument.dimensions[0] == null) { + assertEquals(7L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[1] == null) { + assertEquals(8L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == 0) { + assertEquals(9L, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } } } - assertEquals(7, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 7, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 231 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { @@ -2099,12 +3019,12 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -2112,10 +3032,18 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -2130,19 +3058,50 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { [8, 8] | [8] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[0] == 5) { - assertEquals(starTreeDocument.dimensions[0], starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + if (count <= 10) { + if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[0] == 5) { + assertEquals(starTreeDocument.dimensions[0], starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } } } - assertEquals(10, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 10, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 363 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { @@ -2162,12 +3121,12 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -2175,10 +3134,17 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), getSortedNumericMock(metricsList2, metricsWithField2), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, writeState, mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -2193,18 +3159,49 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { [null, 5] | [5] [null, 7] | [7] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] == null) { - assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + if (count <= 10) { + if (starTreeDocument.dimensions[0] == null) { + assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(10, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 10, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 363 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { @@ -2216,12 +3213,12 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), getSortedNumericMock(metricsList, metricsWithField), - sf, + compositeField, "6" ); @@ -2229,10 +3226,18 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { DocValues.emptySortedNumeric(), DocValues.emptySortedNumeric(), DocValues.emptySortedNumeric(), - sf, + compositeField, "0" ); - builder = getStarTreeBuilder(sf, getWriteState(0), mapperService); + writeState = getWriteState(0, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] @@ -2243,18 +3248,48 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { [4, 4] | [4] [null, 5] | [5] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); int count = 0; - while (starTreeDocumentIterator.hasNext()) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (starTreeDocument.dimensions[0] == null) { - assertEquals(5L, (long) starTreeDocument.dimensions[1]); + if (count <= 6) { + if (starTreeDocument.dimensions[0] == null) { + assertEquals(5L, (long) starTreeDocument.dimensions[1]); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } - assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + compositeField.getMetrics(), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithDuplicateDimensionValues() throws IOException { @@ -2307,7 +3342,7 @@ public void testMergeFlowWithDuplicateDimensionValues() throws IOException { docCountMetricsWithField.add(i); } - StarTreeField sf = getStarTreeFieldWithDocCount(1, true); + compositeField = getStarTreeFieldWithDocCount(1, true); StarTreeValues starTreeValues = getStarTreeValues( dimList1, docsWithField1, @@ -2321,7 +3356,7 @@ public void testMergeFlowWithDuplicateDimensionValues() throws IOException { metricsWithField, docCountMetricsList, docCountMetricsWithField, - sf + compositeField ); StarTreeValues starTreeValues2 = getStarTreeValues( @@ -2337,10 +3372,17 @@ public void testMergeFlowWithDuplicateDimensionValues() throws IOException { metricsWithField, docCountMetricsList, docCountMetricsWithField, - sf + compositeField + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION ); - builder = getStarTreeBuilder(sf, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2))); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); List starTreeDocuments = builder.getStarTreeDocuments(); assertEquals(401, starTreeDocuments.size()); int count = 0; @@ -2371,7 +3413,34 @@ public void testMergeFlowWithDuplicateDimensionValues() throws IOException { count++; } assertEquals(401, count); - validateStarTree(builder.getRootNode(), 4, sf.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 100, + builder.numStarTreeDocs, + 1, + Set.of(), + getBuildMode(), + 0, + 13365 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public void testMergeFlowWithMaxLeafDocs() throws IOException { @@ -2430,7 +3499,7 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { metricsWithField1.add(i); } - StarTreeField sf = getStarTreeFieldWithDocCount(3, true); + compositeField = getStarTreeFieldWithDocCount(3, true); StarTreeValues starTreeValues = getStarTreeValues( dimList1, docsWithField1, @@ -2444,7 +3513,7 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { metricsWithField, metricsList1, metricsWithField1, - sf + compositeField ); StarTreeValues starTreeValues2 = getStarTreeValues( @@ -2460,11 +3529,18 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { metricsWithField, metricsList1, metricsWithField1, - sf + compositeField ); - builder = getStarTreeBuilder(sf, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2))); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); List starTreeDocuments = builder.getStarTreeDocuments(); /** 635 docs get generated @@ -2520,7 +3596,34 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { assertEquals(40L, starTreeDocument.metrics[1]); } } - validateStarTree(builder.getRootNode(), 4, sf.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 100, + builder.numStarTreeDocs, + 3, + Set.of(), + getBuildMode(), + 0, + 23199 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } private StarTreeValues getStarTreeValues( @@ -2544,10 +3647,37 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv, "field5", d3sndv, "field8", d4sndv); - Map metricDocIdSetIterators = new LinkedHashMap<>(); - metricDocIdSetIterators.put("field2", m1sndv); - metricDocIdSetIterators.put("_doc_count", m2sndv); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> d1sndv, + "field3", + () -> d2sndv, + "field5", + () -> d3sndv, + "field8", + () -> d4sndv + ); + + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> m1sndv + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "_doc_count", + sf.getMetrics().get(1).getMetrics().get(0).getTypeName() + ), + () -> m2sndv + ); + // metricDocIdSetIterators.put("field2", () -> m1sndv); + // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); StarTreeValues starTreeValues = new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, getAttributes(500)); return starTreeValues; } @@ -2606,7 +3736,7 @@ public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOE metricsWithField.add(i); } - StarTreeField sf = getStarTreeFieldWithDocCount(3, true); + compositeField = getStarTreeFieldWithDocCount(3, true); StarTreeValues starTreeValues = getStarTreeValues( dimList1, docsWithField1, @@ -2620,7 +3750,7 @@ public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOE metricsWithField, docCountMetricsList, docCountMetricsWithField, - sf + compositeField ); StarTreeValues starTreeValues2 = getStarTreeValues( @@ -2636,13 +3766,48 @@ public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOE metricsWithField, docCountMetricsList, docCountMetricsWithField, - sf + compositeField + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION ); - builder = getStarTreeBuilder(sf, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2))); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); List starTreeDocuments = builder.getStarTreeDocuments(); assertEquals(401, starTreeDocuments.size()); - validateStarTree(builder.getRootNode(), 4, sf.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 100, + builder.numStarTreeDocs, + compositeField.getStarTreeConfig().maxLeafDocs(), + Set.of(), + getBuildMode(), + 0, + 15345 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } public static long getLongFromDouble(double value) { @@ -2714,7 +3879,7 @@ public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOExc metricsList.add(1L); metricsWithField.add(i); } - StarTreeField sf = getStarTreeFieldWithDocCount(10, true); + compositeField = getStarTreeFieldWithDocCount(10, true); StarTreeValues starTreeValues = getStarTreeValues( dimList1, docsWithField1, @@ -2728,7 +3893,7 @@ public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOExc metricsWithField, metricsList1, metricsWithField1, - sf + compositeField ); StarTreeValues starTreeValues2 = getStarTreeValues( @@ -2744,10 +3909,17 @@ public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOExc metricsWithField, metricsList1, metricsWithField1, - sf + compositeField ); - builder = getStarTreeBuilder(sf, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2))); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); List starTreeDocuments = builder.getStarTreeDocuments(); Map> dimValueToDocIdMap = new HashMap<>(); traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); @@ -2762,10 +3934,38 @@ public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOExc } } assertEquals(1041, starTreeDocuments.size()); - validateStarTree(builder.getRootNode(), 4, sf.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 500, + builder.numStarTreeDocs, + compositeField.getStarTreeConfig().maxLeafDocs(), + Set.of(), + getBuildMode(), + 0, + 31779 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } - private static StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { + private StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); Dimension d3 = new NumericDimension("field5"); @@ -2777,11 +3977,7 @@ private static StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boole m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); } List metrics = m2 == null ? List.of(m1) : List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - maxLeafDocs, - new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP - ); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(maxLeafDocs, new HashSet<>(), getBuildMode()); StarTreeField sf = new StarTreeField("sf", dims, metrics, c); return sf; } @@ -2795,23 +3991,24 @@ private void traverseStarTree(InMemoryTreeNode root, Map docIds = new ArrayList<>(); while ((starTreeNode = queue.poll()) != null) { - int dimensionId = starTreeNode.dimensionId; + int dimensionId = starTreeNode.getDimensionId(); if (dimensionId > currentDimensionId) { currentDimensionId = dimensionId; } // store aggregated document of the node - int docId = starTreeNode.aggregatedDocId; + int docId = starTreeNode.getAggregatedDocId(); Map map = dimValueToDocIdMap.getOrDefault(dimensionId, new HashMap<>()); - if (starTreeNode.nodeType == StarTreeNodeType.STAR.getValue()) { + if (starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue()) { map.put(Long.MAX_VALUE, docId); } else { - map.put(starTreeNode.dimensionValue, docId); + map.put(starTreeNode.getDimensionValue(), docId); } dimValueToDocIdMap.put(dimensionId, map); - if (starTreeNode.children != null && (!traverStarNodes || starTreeNode.nodeType == StarTreeNodeType.STAR.getValue())) { - Iterator childrenIterator = starTreeNode.children.values().iterator(); + if (starTreeNode.getChildren() != null + && (!traverStarNodes || starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue())) { + Iterator childrenIterator = starTreeNode.getChildren().values().iterator(); while (childrenIterator.hasNext()) { InMemoryTreeNode childNode = childrenIterator.next(); queue.add(childNode); @@ -2872,21 +4069,34 @@ public void testMergeFlow() throws IOException { Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); List dims = List.of(d1, d2, d3, d4); List metrics = List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1, - new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP - ); - StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv, "field5", d3sndv, "field8", d4sndv); - Map metricDocIdSetIterators = Map.of("field2", m1sndv, "_doc_count", m2sndv); - StarTreeValues starTreeValues = new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, getAttributes(1000)); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> d1sndv, + "field3", + () -> d2sndv, + "field5", + () -> d3sndv, + "field8", + () -> d4sndv + ); + + Map> metricDocIdSetIterators = Map.of("field2", () -> m1sndv, "_doc_count", () -> m2sndv); + + StarTreeValues starTreeValues = new StarTreeValues( + compositeField, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(1000) + ); SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); SortedNumericDocValues f2d2sndv = getSortedNumericMock(dimList2, docsWithField2); @@ -2894,26 +4104,34 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map f2dimDocIdSetIterators = Map.of( + Map> f2dimDocIdSetIterators = Map.of( "field1", - f2d1sndv, + () -> f2d1sndv, "field3", - f2d2sndv, + () -> f2d2sndv, "field5", - f2d3sndv, + () -> f2d3sndv, "field8", - f2d4sndv + () -> f2d4sndv ); - Map f2metricDocIdSetIterators = Map.of("field2", f2m1sndv, "_doc_count", f2m2sndv); + + Map> f2metricDocIdSetIterators = Map.of("field2", () -> f2m1sndv, "_doc_count", () -> f2m2sndv); StarTreeValues starTreeValues2 = new StarTreeValues( - sf, + compositeField, null, f2dimDocIdSetIterators, f2metricDocIdSetIterators, getAttributes(1000) ); - builder = getStarTreeBuilder(sf, writeState, mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** [0, 0, 0, 0] | [0.0, 2] @@ -2929,10 +4147,38 @@ public void testMergeFlow() throws IOException { assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); assertEquals(2L, starTreeDocument.metrics[1]); } - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); // Validate the star tree structure validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 1000, + builder.numStarTreeDocs, + compositeField.getStarTreeConfig().maxLeafDocs(), + Set.of(), + getBuildMode(), + 0, + 132165 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } private void validateStarTree( @@ -2951,36 +4197,37 @@ private void validateStarTree( assertNotNull(node); // assert dimensions - if (node.dimensionId != StarTreeUtils.ALL) { - assertTrue(node.dimensionId >= 0 && node.dimensionId < totalDimensions); + if (node.getDimensionId() != StarTreeUtils.ALL) { + assertTrue(node.getDimensionId() >= 0 && node.getDimensionId() < totalDimensions); } - if (node.children != null && !node.children.isEmpty()) { - assertEquals(node.dimensionId + 1, node.childDimensionId); - assertTrue(node.childDimensionId < totalDimensions); + + if (node.getChildren() != null && !node.getChildren().isEmpty()) { + assertEquals(node.getDimensionId() + 1, node.getChildDimensionId()); + assertTrue(node.getChildDimensionId() < totalDimensions); InMemoryTreeNode starNode = null; Object[] nonStarNodeCumulativeMetrics = getMetrics(starTreeDocuments); - for (Map.Entry entry : node.children.entrySet()) { + for (Map.Entry entry : node.getChildren().entrySet()) { Long childDimensionValue = entry.getKey(); InMemoryTreeNode child = entry.getValue(); Object[] currMetrics = getMetrics(starTreeDocuments); - if (child.nodeType != StarTreeNodeType.STAR.getValue()) { + if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { // Validate dimension values in documents - for (int i = child.startDocId; i < child.endDocId; i++) { + for (int i = child.getStartDocId(); i < child.getEndDocId(); i++) { StarTreeDocument doc = starTreeDocuments.get(i); int j = 0; addMetrics(doc, currMetrics, j); - if (child.nodeType != StarTreeNodeType.STAR.getValue()) { - Long dimension = doc.dimensions[child.dimensionId]; + if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { + Long dimension = doc.dimensions[child.getDimensionId()]; assertEquals(childDimensionValue, dimension); if (dimension != null) { - assertEquals(child.dimensionValue, (long) dimension); + assertEquals(child.getDimensionValue(), (long) dimension); } else { // TODO : fix this ? - assertEquals(child.dimensionValue, StarTreeUtils.ALL); + assertEquals(child.getDimensionValue(), StarTreeUtils.ALL); } } } - Object[] aggregatedMetrics = starTreeDocuments.get(child.aggregatedDocId).metrics; + Object[] aggregatedMetrics = starTreeDocuments.get(child.getAggregatedDocId()).metrics; int j = 0; for (Object metric : currMetrics) { /* @@ -3006,13 +4253,13 @@ private void validateStarTree( // Add star node to queue if (starNode != null) { Object[] starNodeMetrics = getMetrics(starTreeDocuments); - for (int i = starNode.startDocId; i < starNode.endDocId; i++) { + for (int i = starNode.getStartDocId(); i < starNode.getEndDocId(); i++) { StarTreeDocument doc = starTreeDocuments.get(i); int j = 0; addMetrics(doc, starNodeMetrics, j); } int j = 0; - Object[] aggregatedMetrics = starTreeDocuments.get(starNode.aggregatedDocId).metrics; + Object[] aggregatedMetrics = starTreeDocuments.get(starNode.getAggregatedDocId()).metrics; for (Object nonStarNodeCumulativeMetric : nonStarNodeCumulativeMetrics) { assertEquals(nonStarNodeCumulativeMetric, starNodeMetrics[j]); assertEquals(starNodeMetrics[j], aggregatedMetrics[j]); @@ -3032,20 +4279,20 @@ private void validateStarTree( j++; } - assertEquals(-1L, starNode.dimensionValue); + assertEquals(-1L, starNode.getDimensionValue()); queue.offer(new Object[] { starNode, true }); } } else { - assertTrue(node.endDocId - node.startDocId <= maxLeafDocuments); + assertTrue(node.getEndDocId() - node.getStartDocId() <= maxLeafDocuments); } if (currentIsStarNode) { StarTreeDocument prevDoc = null; int docCount = 0; - int docId = node.startDocId; - int dimensionId = node.dimensionId; + int docId = node.getStartDocId(); + int dimensionId = node.getDimensionId(); - while (docId < node.endDocId) { + while (docId < node.getEndDocId()) { StarTreeDocument currentDoc = starTreeDocuments.get(docId); docCount++; @@ -3061,7 +4308,7 @@ private void validateStarTree( } // Verify that the number of generated star documents matches the range in the star node - assertEquals(node.endDocId - node.startDocId, docCount); + assertEquals(node.getEndDocId() - node.getStartDocId(), docCount); } } } @@ -3111,20 +4358,16 @@ private int compareDocuments(StarTreeDocument doc1, StarTreeDocument doc2, int s } Map getAttributes(int numSegmentDocs) { - return Map.of(String.valueOf(NUM_SEGMENT_DOCS), String.valueOf(numSegmentDocs)); + return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); } - private static StarTreeField getStarTreeField(MetricStat count) { + private StarTreeField getStarTreeField(MetricStat count) { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(count)); List dims = List.of(d1, d2); List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1000, - new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP - ); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); return new StarTreeField("sf", dims, metrics, c); } @@ -3179,6 +4422,9 @@ public void tearDown() throws Exception { if (builder != null) { builder.close(); } + docValuesConsumer.close(); + metaOut.close(); + dataOut.close(); directory.close(); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 51ebc02ea8243..22a0fb7875714 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -8,21 +8,24 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -31,6 +34,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; @@ -51,6 +55,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -75,9 +80,12 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static List metrics; private static Directory directory; private static FieldInfo[] fieldsInfo; - private static SegmentWriteState state; + private static SegmentWriteState writeState; private static StarTreeField starTreeField; + private static IndexOutput dataOut; + private static IndexOutput metaOut; + @BeforeClass public static void setup() throws IOException { @@ -138,7 +146,21 @@ public static void setup() throws IOException { fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - state = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + + String dataFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.DATA_EXTENSION + ); + dataOut = writeState.directory.createOutput(dataFileName, writeState.context); + + String metaFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.META_EXTENSION + ); + metaOut = writeState.directory.createOutput(metaFileName, writeState.context); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -157,9 +179,13 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(starTreeField, state, mapperService) { + builder = new BaseStarTreeBuilder(metaOut, dataOut, starTreeField, writeState, mapperService) { @Override - public void build(List starTreeValuesSubs) throws IOException {} + public void build( + List starTreeValuesSubs, + AtomicInteger fieldNumberAcrossStarTrees, + DocValuesConsumer starTreeDocValuesConsumer + ) throws IOException {} @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -224,6 +250,8 @@ public void test_reduceStarTreeDocuments() { @Override public void tearDown() throws Exception { super.tearDown(); + dataOut.close(); + metaOut.close(); directory.close(); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java index 92382b78f60c6..496558dbc2e83 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java @@ -9,7 +9,9 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -17,10 +19,18 @@ public class OffHeapStarTreeBuilderTests extends AbstractStarTreeBuilderTests { @Override public BaseStarTreeBuilder getStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, StarTreeField starTreeField, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - return new OffHeapStarTreeBuilder(starTreeField, segmentWriteState, mapperService); + return new OffHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); } + + @Override + StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode() { + return StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP; + } + } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index aed08b7727be7..55cf3bde3cea7 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -9,16 +9,29 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.mapper.MapperService; +import java.io.IOException; + public class OnHeapStarTreeBuilderTests extends AbstractStarTreeBuilderTests { + @Override public BaseStarTreeBuilder getStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, StarTreeField starTreeField, SegmentWriteState segmentWriteState, MapperService mapperService - ) { - return new OnHeapStarTreeBuilder(starTreeField, segmentWriteState, mapperService); + ) throws IOException { + return new OnHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); + } + + @Override + StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode() { + return StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP; } + } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java index 828bddfb8aa6e..94fe148eab6b3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.FieldInfo; @@ -15,6 +16,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -31,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verifyNoInteractions; @@ -45,9 +48,13 @@ public class StarTreesBuilderTests extends OpenSearchTestCase { private StarTreeField starTreeField; private Map fieldProducerMap; private Directory directory; + private IndexOutput dataOut; + private IndexOutput metaOut; public void setUp() throws Exception { super.setUp(); + metaOut = mock(IndexOutput.class); + dataOut = mock(IndexOutput.class); mapperService = mock(MapperService.class); directory = newFSDirectory(createTempDir()); SegmentInfo segmentInfo = new SegmentInfo( @@ -88,19 +95,29 @@ public void setUp() throws Exception { public void test_buildWithNoStarTreeFields() throws IOException { when(mapperService.getCompositeFieldTypes()).thenReturn(new HashSet<>()); - StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService); - starTreesBuilder.build(fieldProducerMap); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger()); + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, mock(DocValuesConsumer.class)); verifyNoInteractions(docValuesProducer); } public void test_getStarTreeBuilder() throws IOException { when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); - StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService); - StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(starTreeField, segmentWriteState, mapperService); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger()); + StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); assertTrue(starTreeBuilder instanceof OnHeapStarTreeBuilder); } + public void test_getStarTreeBuilder_illegalArgument() throws IOException { + when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); + StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP); + StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger()); + StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); + assertTrue(starTreeBuilder instanceof OffHeapStarTreeBuilder); + starTreeBuilder.close(); + } + public void test_closeWithNoStarTreeFields() throws IOException { StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration( 1, @@ -110,7 +127,7 @@ public void test_closeWithNoStarTreeFields() throws IOException { StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); starTreeFieldType = new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField); when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); - StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger()); starTreesBuilder.close(); verifyNoInteractions(docValuesProducer); @@ -119,6 +136,8 @@ public void test_closeWithNoStarTreeFields() throws IOException { @Override public void tearDown() throws Exception { super.tearDown(); + metaOut.close(); + dataOut.close(); directory.close(); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java index 4653ac8b08198..4d2aa5eaf78cf 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.ArrayDeque; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -121,8 +120,8 @@ public void test_starTreeSearch() throws IOException { assertNotNull(inMemoryTreeNode); for (int i = 0; i < maxLevels - 1; i++) { - InMemoryTreeNode randomChildNode = randomFrom(inMemoryTreeNode.children.values()); - StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue(randomChildNode.dimensionValue); + InMemoryTreeNode randomChildNode = randomFrom(inMemoryTreeNode.getChildren().values()); + StarTreeNode randomStarTreeChildNode = starTreeNode.getChildForDimensionValue(randomChildNode.getDimensionValue()); assertNotNull(randomStarTreeChildNode); assertStarTreeNode(randomStarTreeChildNode, randomChildNode); @@ -135,18 +134,18 @@ public void test_starTreeSearch() throws IOException { } private void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { - assertEquals(starTreeNode.getDimensionId(), treeNode.dimensionId); - assertEquals(starTreeNode.getDimensionValue(), treeNode.dimensionValue); - assertEquals(starTreeNode.getStartDocId(), treeNode.startDocId); - assertEquals(starTreeNode.getEndDocId(), treeNode.endDocId); - assertEquals(starTreeNode.getChildDimensionId(), treeNode.childDimensionId); - assertEquals(starTreeNode.getAggregatedDocId(), treeNode.aggregatedDocId); - assertEquals(starTreeNode.getStarTreeNodeType(), treeNode.nodeType); + assertEquals(starTreeNode.getDimensionId(), treeNode.getDimensionId()); + assertEquals(starTreeNode.getDimensionValue(), treeNode.getDimensionValue()); + assertEquals(starTreeNode.getStartDocId(), treeNode.getStartDocId()); + assertEquals(starTreeNode.getEndDocId(), treeNode.getEndDocId()); + assertEquals(starTreeNode.getChildDimensionId(), treeNode.getChildDimensionId()); + assertEquals(starTreeNode.getAggregatedDocId(), treeNode.getAggregatedDocId()); + assertEquals(starTreeNode.getStarTreeNodeType(), treeNode.getNodeType()); if (starTreeNode.getChildDimensionId() != -1) { assertFalse(starTreeNode.isLeaf()); - if (treeNode.children != null) { - assertEquals(starTreeNode.getNumChildren(), treeNode.children.values().size()); + if (treeNode.getChildren() != null) { + assertEquals(starTreeNode.getNumChildren(), treeNode.getChildren().values().size()); } } else { assertTrue(starTreeNode.isLeaf()); @@ -156,16 +155,11 @@ private void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode tree public InMemoryTreeNode generateSampleTree(Map inMemoryTreeNodeMap) { // Create the root node - InMemoryTreeNode root = new InMemoryTreeNode(); - root.dimensionId = 0; - root.startDocId = randomInt(); - root.endDocId = randomInt(); - root.childDimensionId = 1; - root.aggregatedDocId = randomInt(); - root.nodeType = (byte) 0; - root.children = new HashMap<>(); + InMemoryTreeNode root = new InMemoryTreeNode(0, randomInt(), randomInt(), (byte) 0, -1); + root.setChildDimensionId(1); + root.setAggregatedDocId(randomInt()); - inMemoryTreeNodeMap.put(root.dimensionValue, root); + inMemoryTreeNodeMap.put(root.getDimensionValue(), root); // Generate the tree recursively generateTreeRecursively(root, 1, inMemoryTreeNodeMap); @@ -181,19 +175,14 @@ private void generateTreeRecursively(InMemoryTreeNode parent, int currentLevel, int numChildren = randomIntBetween(1, 10); for (int i = 0; i < numChildren; i++) { - InMemoryTreeNode child = new InMemoryTreeNode(); dimensionValue++; - child.dimensionId = currentLevel; - child.dimensionValue = dimensionValue; // Assign a unique dimension value for each child - child.startDocId = randomInt(); - child.endDocId = randomInt(); - child.childDimensionId = (currentLevel == this.maxLevels - 1) ? -1 : (currentLevel + 1); - child.aggregatedDocId = randomInt(); - child.nodeType = (byte) 0; - child.children = new HashMap<>(); - - parent.children.put(child.dimensionValue, child); - inMemoryTreeNodeMap.put(child.dimensionValue, child); + InMemoryTreeNode child = new InMemoryTreeNode(currentLevel, randomInt(), randomInt(), (byte) 0, dimensionValue); + + child.setChildDimensionId((currentLevel == this.maxLevels - 1) ? -1 : (currentLevel + 1)); + child.setAggregatedDocId(randomInt()); + + parent.addChildNode(child, child.getDimensionValue()); + inMemoryTreeNodeMap.put(child.getDimensionValue(), child); generateTreeRecursively(child, currentLevel + 1, inMemoryTreeNodeMap); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index 62bd74cc0b3fc..6fac5362c3dbb 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -62,6 +62,7 @@ public class StarTreeMetadataTests extends OpenSearchTestCase { private List metrics; private List metricAggregatorInfos = new ArrayList<>(); private int segmentDocumentCount; + private int numStarTreeDocs; private long dataFilePointer; private long dataFileLength; @@ -145,6 +146,7 @@ public void test_starTreeMetadata() throws IOException { dataFileLength = randomNonNegativeLong(); dataFilePointer = randomNonNegativeLong(); segmentDocumentCount = randomInt(Integer.MAX_VALUE); + numStarTreeDocs = randomInt(Integer.MAX_VALUE); metaOut = directory.createOutput("star-tree-metadata", IOContext.DEFAULT); StarTreeWriter starTreeWriter = new StarTreeWriter(); int numberOfNodes = randomInt(Integer.MAX_VALUE); @@ -154,6 +156,7 @@ public void test_starTreeMetadata() throws IOException { metricAggregatorInfos, numberOfNodes, segmentDocumentCount, + numStarTreeDocs, dataFilePointer, dataFileLength ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index 6f24728c24f30..9657f867422a0 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -12,18 +12,18 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; import java.io.IOException; -import java.util.HashMap; import java.util.Iterator; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,6 +35,7 @@ public class FixedLengthStarTreeNodeTests extends OpenSearchTestCase { InMemoryTreeNode node; InMemoryTreeNode starChild; InMemoryTreeNode nullChild; + InMemoryTreeNode childWithMinus1; FixedLengthStarTreeNode starTreeNode; @Before @@ -44,54 +45,42 @@ public void setup() throws IOException { dataOut = directory.createOutput("star-tree-data", IOContext.DEFAULT); StarTreeWriter starTreeWriter = new StarTreeWriter(); - node = new InMemoryTreeNode(); - node.dimensionId = 0; - node.startDocId = randomInt(); - node.endDocId = randomInt(); - node.childDimensionId = 1; - node.aggregatedDocId = randomInt(); - node.nodeType = randomFrom((byte) 0, (byte) -1, (byte) 2); - node.children = new HashMap<>(); - - starChild = new InMemoryTreeNode(); - starChild.dimensionId = node.dimensionId + 1; - starChild.dimensionValue = -1; - starChild.startDocId = randomInt(); - starChild.endDocId = randomInt(); - starChild.childDimensionId = -1; - starChild.aggregatedDocId = randomInt(); - starChild.nodeType = (byte) -2; - starChild.children = new HashMap<>(); - node.children.put(-1L, starChild); - - nullChild = new InMemoryTreeNode(); - nullChild.dimensionId = node.dimensionId + 1; - nullChild.dimensionValue = -1; - nullChild.startDocId = randomInt(); - nullChild.endDocId = randomInt(); - nullChild.childDimensionId = -1; - nullChild.aggregatedDocId = randomInt(); - nullChild.nodeType = (byte) -1; - nullChild.children = new HashMap<>(); - node.children.put(null, nullChild); + node = new InMemoryTreeNode(0, randomInt(), randomInt(), randomFrom((byte) 0, (byte) -1, (byte) 1), -1); + node.setChildDimensionId(1); + node.setAggregatedDocId(randomInt()); + + starChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) -1, -1); + starChild.setChildDimensionId(-1); + starChild.setAggregatedDocId(randomInt()); + node.addChildNode(starChild, (long) ALL); + + childWithMinus1 = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 0, -1); + childWithMinus1.setChildDimensionId(-1); + childWithMinus1.setAggregatedDocId(randomInt()); + node.addChildNode(childWithMinus1, -1L); for (int i = 1; i < randomIntBetween(2, 5); i++) { - InMemoryTreeNode child = new InMemoryTreeNode(); - child.dimensionId = node.dimensionId + 1; - child.dimensionValue = node.dimensionValue + i; // Assign a unique dimension value for each child - child.startDocId = randomInt(); - child.endDocId = randomInt(); - child.childDimensionId = -1; - child.aggregatedDocId = randomInt(); - child.nodeType = (byte) 0; - child.children = new HashMap<>(); - node.children.put(child.dimensionValue, child); + InMemoryTreeNode child = new InMemoryTreeNode( + node.getDimensionId() + 1, + randomInt(), + randomInt(), + (byte) 0, + node.getDimensionValue() + i + ); + child.setChildDimensionId(-1); + child.setAggregatedDocId(randomInt()); + node.addChildNode(child, child.getDimensionValue()); } - long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 1 + node.children.size(), "star-tree"); + nullChild = new InMemoryTreeNode(node.getDimensionId() + 1, randomInt(), randomInt(), (byte) 1, -1); + nullChild.setChildDimensionId(-1); + nullChild.setAggregatedDocId(randomInt()); + node.addChildNode(nullChild, null); + + long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 2 + node.getChildren().size(), "star-tree"); // asserting on the actual length of the star tree data file - assertEquals(starTreeDataLength, 33L * node.children.size() + 33); + assertEquals(starTreeDataLength, 33L * node.getChildren().size() + 2 * 33); dataOut.close(); dataIn = directory.openInput("star-tree-data", IOContext.READONCE); @@ -119,27 +108,27 @@ public void testSerializableDataSize() { } public void testGetDimensionId() throws IOException { - assertEquals(node.dimensionId, starTreeNode.getDimensionId()); + assertEquals(node.getDimensionId(), starTreeNode.getDimensionId()); } public void testGetDimensionValue() throws IOException { - assertEquals(node.dimensionValue, starTreeNode.getDimensionValue()); + assertEquals(node.getDimensionValue(), starTreeNode.getDimensionValue()); } public void testGetStartDocId() throws IOException { - assertEquals(node.startDocId, starTreeNode.getStartDocId()); + assertEquals(node.getStartDocId(), starTreeNode.getStartDocId()); } public void testGetEndDocId() throws IOException { - assertEquals(node.endDocId, starTreeNode.getEndDocId()); + assertEquals(node.getEndDocId(), starTreeNode.getEndDocId()); } public void testGetAggregatedDocId() throws IOException { - assertEquals(node.aggregatedDocId, starTreeNode.getAggregatedDocId()); + assertEquals(node.getAggregatedDocId(), starTreeNode.getAggregatedDocId()); } public void testGetNumChildren() throws IOException { - assertEquals(node.children.size(), starTreeNode.getNumChildren()); + assertEquals(node.getChildren().size(), starTreeNode.getNumChildren() - 1); } public void testIsLeaf() { @@ -151,8 +140,7 @@ public void testGetStarTreeNodeType() throws IOException { } public void testGetChildForDimensionValue() throws IOException { - // TODO: Add a test to verify children with star node, null node and default node with default dimension value -1 - long dimensionValue = randomIntBetween(0, node.children.size() - 3); + long dimensionValue = randomIntBetween(-1, node.getChildren().size() - 3); FixedLengthStarTreeNode childNode = (FixedLengthStarTreeNode) starTreeNode.getChildForDimensionValue(dimensionValue); assertNotNull(childNode); assertEquals(dimensionValue, childNode.getDimensionValue()); @@ -169,11 +157,31 @@ public void testGetChildrenIterator() throws IOException { assertEquals(starTreeNode.getNumChildren(), count); } + public void testChildrenOrder() throws IOException { + Iterator iterator = starTreeNode.getChildrenIterator(); + int count = 0; + while (iterator.hasNext()) { + FixedLengthStarTreeNode child = iterator.next(); + count++; + if (count == 1) { + StarTreeTestUtils.assertStarTreeNode(child, starChild); + } else if (count == 2) { + StarTreeTestUtils.assertStarTreeNode(child, childWithMinus1); + } else if (count == starTreeNode.getNumChildren()) { + StarTreeTestUtils.assertStarTreeNode(child, nullChild); + } else { + StarTreeTestUtils.assertStarTreeNode(child, node.getChildren().get(child.getDimensionValue())); + } + assertNotNull(child); + } + assertEquals(starTreeNode.getNumChildren(), count); + } + public void testGetChildForStarNode() throws IOException { // Assuming the first child is a star node in our test data FixedLengthStarTreeNode starNode = (FixedLengthStarTreeNode) starTreeNode.getChildStarNode(); assertNotNull(starNode); - assertEquals(StarTreeUtils.ALL, starNode.getDimensionValue()); + assertEquals(ALL, starNode.getDimensionValue()); } public void testGetChildForNullNode() throws IOException { @@ -193,14 +201,9 @@ public void testOnlyRootNodePresent() throws IOException { IndexOutput dataOut = directory.createOutput("star-tree-data-1", IOContext.DEFAULT); StarTreeWriter starTreeWriter = new StarTreeWriter(); - InMemoryTreeNode node = new InMemoryTreeNode(); - node.dimensionId = 0; - node.startDocId = randomInt(); - node.endDocId = randomInt(); - node.childDimensionId = 1; - node.aggregatedDocId = randomInt(); - node.nodeType = randomFrom((byte) 0, (byte) -1, (byte) 2); - node.children = new HashMap<>(); + InMemoryTreeNode node = new InMemoryTreeNode(0, randomInt(), randomInt(), randomFrom((byte) 0, (byte) -1, (byte) 2), -1); + node.setChildDimensionId(1); + node.setAggregatedDocId(randomInt()); long starTreeDataLength = starTreeWriter.writeStarTree(dataOut, node, 1, "star-tree"); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeTypeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeTypeTests.java index 81fb620da5af3..f1034e5d1d033 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeTypeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNodeTypeTests.java @@ -14,12 +14,12 @@ public class StarTreeNodeTypeTests extends OpenSearchTestCase { public void testStarNodeType() { assertEquals("star", StarTreeNodeType.STAR.getName()); - assertEquals((byte) -2, StarTreeNodeType.STAR.getValue()); + assertEquals((byte) -1, StarTreeNodeType.STAR.getValue()); } public void testNullNodeType() { assertEquals("null", StarTreeNodeType.NULL.getName()); - assertEquals((byte) -1, StarTreeNodeType.NULL.getValue()); + assertEquals((byte) 1, StarTreeNodeType.NULL.getValue()); } public void testDefaultNodeType() { @@ -28,20 +28,20 @@ public void testDefaultNodeType() { } public void testFromValue() { - assertEquals(StarTreeNodeType.STAR, StarTreeNodeType.fromValue((byte) -2)); - assertEquals(StarTreeNodeType.NULL, StarTreeNodeType.fromValue((byte) -1)); + assertEquals(StarTreeNodeType.STAR, StarTreeNodeType.fromValue((byte) -1)); + assertEquals(StarTreeNodeType.NULL, StarTreeNodeType.fromValue((byte) 1)); assertEquals(StarTreeNodeType.DEFAULT, StarTreeNodeType.fromValue((byte) 0)); } public void testFromValueInvalid() { - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> StarTreeNodeType.fromValue((byte) 1)); - assertEquals("Unrecognized value byte to determine star-tree node type: [1]", exception.getMessage()); + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> StarTreeNodeType.fromValue((byte) 2)); + assertEquals("Unrecognized value byte to determine star-tree node type: [2]", exception.getMessage()); } public void testEnumValues() { StarTreeNodeType[] values = StarTreeNodeType.values(); assertEquals(3, values.length); - assertArrayEquals(new StarTreeNodeType[] { StarTreeNodeType.STAR, StarTreeNodeType.NULL, StarTreeNodeType.DEFAULT }, values); + assertArrayEquals(new StarTreeNodeType[] { StarTreeNodeType.STAR, StarTreeNodeType.DEFAULT, StarTreeNodeType.NULL }, values); } public void testEnumValueOf() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java index a1d341615969e..9cca0b04e9ea4 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java @@ -52,7 +52,7 @@ public void testGetFieldInfoList() { public void testGetFieldInfo() { String fieldName = UUID.randomUUID().toString(); int fieldNumber = randomInt(); - assertFieldInfos(StarTreeUtils.getFieldInfo(fieldName, fieldNumber), fieldName, fieldNumber); + assertFieldInfos(StarTreeUtils.getFieldInfo(fieldName, DocValuesType.SORTED_NUMERIC, fieldNumber), fieldName, fieldNumber); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 6afc7c23d9e66..e07d7ef3c5a7b 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -815,6 +815,14 @@ public static long randomNonNegativeLong() { return randomLong == Long.MIN_VALUE ? 0 : Math.abs(randomLong); } + /** + * @return a int between 0 and Integer.MAX_VALUE (inclusive) chosen uniformly at random. + */ + public static int randomNonNegativeInt() { + int randomInt = randomInt(); + return randomInt == Integer.MIN_VALUE ? 0 : Math.abs(randomInt); + } + public static float randomFloat() { return random().nextFloat(); } From 724726648df24a4c379f642401b218e83d8fee61 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Mon, 2 Sep 2024 21:28:32 +0530 Subject: [PATCH 007/357] Simplify diff calculation for remote routing table (#15100) Simplify diff calculation logic for remote routing table Signed-off-by: Bukhtawar Khan Co-authored-by: Shailendra Singh --- .../remote/RemoteRoutingTableServiceIT.java | 89 ++++- .../routing/IndexShardRoutingTable.java | 18 +- .../cluster/routing/RoutingTable.java | 11 +- .../routing/RoutingTableIncrementalDiff.java | 184 ++++------ .../routing/StringKeyDiffProvider.java | 33 ++ .../InternalRemoteRoutingTableService.java | 28 +- .../remote/NoopRemoteRoutingTableService.java | 19 +- .../remote/RemoteRoutingTableService.java | 53 +-- .../remote/ClusterMetadataManifest.java | 26 +- .../remote/ClusterStateDiffManifest.java | 61 ++-- .../remote/RemoteClusterStateService.java | 81 +++-- .../model/RemoteClusterMetadataManifest.java | 5 + .../routingtable/RemoteRoutingTableDiff.java | 24 +- .../routing/RoutingTableDiffTests.java | 325 ++++++++++++++++++ .../RemoteRoutingTableServiceTests.java | 169 ++++----- .../remote/ClusterMetadataManifestTests.java | 78 ++++- .../RemoteClusterStateServiceTests.java | 25 +- .../model/ClusterStateDiffManifestTests.java | 73 +++- .../RemoteIndexRoutingTableDiffTests.java | 190 ++++------ .../cluster/OpenSearchAllocationTestCase.java | 13 + 20 files changed, 975 insertions(+), 530 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/StringKeyDiffProvider.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/RoutingTableDiffTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java index b0d046cbdf3db..9b80e4f907070 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java @@ -24,6 +24,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.Before; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; @@ -46,7 +47,7 @@ public class RemoteRoutingTableServiceIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "test-index"; private static final String INDEX_NAME_1 = "test-index-1"; - BlobPath indexRoutingPath; + List indexRoutingPaths; AtomicInteger indexRoutingFiles = new AtomicInteger(); private final RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.HASHED_PREFIX; @@ -91,7 +92,7 @@ public void testRemoteRoutingTableIndexLifecycle() throws Exception { updateIndexSettings(INDEX_NAME, IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2); ensureGreen(INDEX_NAME); assertBusy(() -> { - int indexRoutingFilesAfterUpdate = repository.blobStore().blobContainer(indexRoutingPath).listBlobs().size(); + int indexRoutingFilesAfterUpdate = repository.blobStore().blobContainer(indexRoutingPaths.get(0)).listBlobs().size(); // At-least 3 new index routing files will be created as shards will transition from INIT -> UNASSIGNED -> STARTED state assertTrue(indexRoutingFilesAfterUpdate >= indexRoutingFiles.get() + 3); }); @@ -112,6 +113,47 @@ public void testRemoteRoutingTableIndexLifecycle() throws Exception { assertTrue(areRoutingTablesSame(routingTableVersions)); } + public void testRemoteRoutingTableWithMultipleIndex() throws Exception { + BlobStoreRepository repository = prepareClusterAndVerifyRepository(); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + RemoteManifestManager remoteManifestManager = remoteClusterStateService.getRemoteManifestManager(); + Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + List expectedIndexNames = new ArrayList<>(); + List deletedIndexNames = new ArrayList<>(); + verifyUpdatesInManifestFile(latestManifest, expectedIndexNames, 1, deletedIndexNames, true); + + List routingTables = getRoutingTableFromAllNodes(); + // Verify indices in routing table + Set expectedIndicesInRoutingTable = Set.of(INDEX_NAME); + assertEquals(routingTables.get(0).getIndicesRouting().keySet(), expectedIndicesInRoutingTable); + // Verify routing table across all nodes is equal + assertTrue(areRoutingTablesSame(routingTables)); + + // Create new index + createIndex(INDEX_NAME_1, remoteStoreIndexSettings(1, 5)); + ensureGreen(INDEX_NAME_1); + + latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + + updateIndexRoutingPaths(repository); + verifyUpdatesInManifestFile(latestManifest, expectedIndexNames, 2, deletedIndexNames, true); + routingTables = getRoutingTableFromAllNodes(); + // Verify indices in routing table + expectedIndicesInRoutingTable = Set.of(INDEX_NAME, INDEX_NAME_1); + assertEquals(routingTables.get(0).getIndicesRouting().keySet(), expectedIndicesInRoutingTable); + // Verify routing table across all nodes is equal + assertTrue(areRoutingTablesSame(routingTables)); + } + public void testRemoteRoutingTableEmptyRoutingTableDiff() throws Exception { prepareClusterAndVerifyRepository(); @@ -166,7 +208,7 @@ public void testRemoteRoutingTableIndexNodeRestart() throws Exception { assertRemoteStoreRepositoryOnAllNodes(REMOTE_ROUTING_TABLE_REPO); assertBusy(() -> { - int indexRoutingFilesAfterNodeDrop = repository.blobStore().blobContainer(indexRoutingPath).listBlobs().size(); + int indexRoutingFilesAfterNodeDrop = repository.blobStore().blobContainer(indexRoutingPaths.get(0)).listBlobs().size(); assertTrue(indexRoutingFilesAfterNodeDrop > indexRoutingFiles.get()); }); @@ -201,7 +243,7 @@ public void testRemoteRoutingTableIndexMasterRestart() throws Exception { assertRemoteStoreRepositoryOnAllNodes(REMOTE_ROUTING_TABLE_REPO); assertBusy(() -> { - int indexRoutingFilesAfterNodeDrop = repository.blobStore().blobContainer(indexRoutingPath).listBlobs().size(); + int indexRoutingFilesAfterNodeDrop = repository.blobStore().blobContainer(indexRoutingPaths.get(0)).listBlobs().size(); assertTrue(indexRoutingFilesAfterNodeDrop > indexRoutingFiles.get()); }); @@ -240,10 +282,14 @@ private BlobStoreRepository prepareClusterAndVerifyRepository() throws Exception BlobPath baseMetadataPath = getBaseMetadataPath(repository); List indexRoutingTables = new ArrayList<>(getClusterState().routingTable().indicesRouting().values()); - indexRoutingPath = getIndexRoutingPath(baseMetadataPath.add(INDEX_ROUTING_TABLE), indexRoutingTables.get(0).getIndex().getUUID()); + indexRoutingPaths = new ArrayList<>(); + for (IndexRoutingTable indexRoutingTable : indexRoutingTables) { + indexRoutingPaths.add(getIndexRoutingPath(baseMetadataPath.add(INDEX_ROUTING_TABLE), indexRoutingTable.getIndex().getUUID())); + } assertBusy(() -> { - indexRoutingFiles.set(repository.blobStore().blobContainer(indexRoutingPath).listBlobs().size()); + int totalRoutingFiles = calculateTotalRoutingFiles(repository); + indexRoutingFiles.set(totalRoutingFiles); // There would be >=3 files as shards will transition from UNASSIGNED -> INIT -> STARTED state assertTrue(indexRoutingFiles.get() >= 3); }); @@ -280,11 +326,19 @@ private void verifyUpdatesInManifestFile( assertTrue(latestManifest.isPresent()); ClusterMetadataManifest manifest = latestManifest.get(); - assertEquals(expectedIndexNames, manifest.getDiffManifest().getIndicesRoutingUpdated()); assertEquals(expectedDeletedIndex, manifest.getDiffManifest().getIndicesDeleted()); assertEquals(expectedIndicesRoutingFilesInManifest, manifest.getIndicesRouting().size()); + + // Check if all paths in manifest.getIndicesRouting() are present in indexRoutingPaths for (ClusterMetadataManifest.UploadedIndexMetadata uploadedFilename : manifest.getIndicesRouting()) { - assertTrue(uploadedFilename.getUploadedFilename().contains(indexRoutingPath.buildAsString())); + boolean pathFound = false; + for (BlobPath indexRoutingPath : indexRoutingPaths) { + if (uploadedFilename.getUploadedFilename().contains(indexRoutingPath.buildAsString())) { + pathFound = true; + break; + } + } + assertTrue("Uploaded file not found in indexRoutingPaths: " + uploadedFilename.getUploadedFilename(), pathFound); } assertEquals(isRoutingTableDiffFileExpected, manifest.getDiffManifest().getIndicesRoutingDiffPath() != null); } @@ -305,6 +359,24 @@ private List getRoutingTableFromAllNodes() throws ExecutionExcepti return routingTables; } + private void updateIndexRoutingPaths(BlobStoreRepository repository) { + BlobPath baseMetadataPath = getBaseMetadataPath(repository); + List indexRoutingTables = new ArrayList<>(getClusterState().routingTable().indicesRouting().values()); + + indexRoutingPaths.clear(); // Clear the list to avoid stale data + for (IndexRoutingTable indexRoutingTable : indexRoutingTables) { + indexRoutingPaths.add(getIndexRoutingPath(baseMetadataPath.add(INDEX_ROUTING_TABLE), indexRoutingTable.getIndex().getUUID())); + } + } + + private int calculateTotalRoutingFiles(BlobStoreRepository repository) throws IOException { + int totalRoutingFiles = 0; + for (BlobPath path : indexRoutingPaths) { + totalRoutingFiles += repository.blobStore().blobContainer(path).listBlobs().size(); + } + return totalRoutingFiles; + } + private boolean areRoutingTablesSame(List routingTables) { if (routingTables == null || routingTables.isEmpty()) { return false; @@ -356,7 +428,6 @@ private void deleteIndexAndVerify(RemoteManifestManager remoteManifestManager) { ); assertTrue(latestManifest.isPresent()); ClusterMetadataManifest manifest = latestManifest.get(); - assertTrue(manifest.getDiffManifest().getIndicesRoutingUpdated().isEmpty()); assertTrue(manifest.getDiffManifest().getIndicesDeleted().contains(INDEX_NAME)); assertTrue(manifest.getIndicesRouting().isEmpty()); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 00ddef507a162..f782cfc1b060a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -34,6 +34,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.cluster.Diff; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Nullable; @@ -75,7 +77,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexShardRoutingTable implements Iterable { +public class IndexShardRoutingTable extends AbstractDiffable implements Iterable { final ShardShuffler shuffler; // Shuffler for weighted round-robin shard routing. This uses rotation to permute shards. @@ -545,6 +547,12 @@ private static List rankShardsAndUpdateStats( return sortedShards; } + @Override + public void writeTo(StreamOutput out) throws IOException { + this.shardId().getIndex().writeTo(out); + Builder.writeToThin(this, out); + } + private static class NodeRankComparator implements Comparator { private final Map nodeRanks; @@ -1067,6 +1075,14 @@ private void populateInitializingShardWeightsMap(WeightedRouting weightedRouting } } + public static IndexShardRoutingTable readFrom(StreamInput in) throws IOException { + return IndexShardRoutingTable.Builder.readFrom(in); + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(IndexShardRoutingTable::readFrom, in); + } + /** * Builder of an index shard routing table. * diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index f805ed09c2512..544d4f9326fba 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -378,6 +378,10 @@ public Diff diff(RoutingTable previousState) { return new RoutingTableDiff(previousState, this); } + public Diff incrementalDiff(RoutingTable previousState) { + return new RoutingTableIncrementalDiff(previousState, this); + } + public static Diff readDiffFrom(StreamInput in) throws IOException { return new RoutingTableDiff(in); } @@ -403,7 +407,7 @@ public void writeTo(StreamOutput out) throws IOException { } } - private static class RoutingTableDiff implements Diff { + private static class RoutingTableDiff implements Diff, StringKeyDiffProvider { private final long version; @@ -432,6 +436,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); indicesRouting.writeTo(out); } + + @Override + public DiffableUtils.MapDiff> provideDiff() { + return (DiffableUtils.MapDiff>) indicesRouting; + } } public static Builder builder() { diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTableIncrementalDiff.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTableIncrementalDiff.java index 3d75b22a8ed7f..13501a431d9f9 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTableIncrementalDiff.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTableIncrementalDiff.java @@ -9,93 +9,85 @@ package org.opensearch.cluster.routing; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.Index; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.Map; +import static org.opensearch.cluster.DiffableUtils.MapDiff; + /** * Represents a difference between {@link RoutingTable} objects that can be serialized and deserialized. */ -public class RoutingTableIncrementalDiff implements Diff { +public class RoutingTableIncrementalDiff implements Diff, StringKeyDiffProvider { - private final Map> diffs; + private final Diff> indicesRouting; - /** - * Constructs a new RoutingTableIncrementalDiff with the given differences. - * - * @param diffs a map containing the differences of {@link IndexRoutingTable}. - */ - public RoutingTableIncrementalDiff(Map> diffs) { - this.diffs = diffs; + private final long version; + + private static final DiffableUtils.DiffableValueSerializer CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER = + new DiffableUtils.DiffableValueSerializer<>() { + + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + + @Override + public Diff readDiff(StreamInput in, String key) throws IOException { + return new RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff(in); + } + + @Override + public Diff diff(IndexRoutingTable currentState, IndexRoutingTable previousState) { + return new RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff( + currentState.getIndex(), + previousState, + currentState + ); + } + }; + + public RoutingTableIncrementalDiff(RoutingTable before, RoutingTable after) { + version = after.version(); + indicesRouting = DiffableUtils.diff( + before.getIndicesRouting(), + after.getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER + ); } - /** - * Gets the map of differences of {@link IndexRoutingTable}. - * - * @return a map containing the differences. - */ - public Map> getDiffs() { - return diffs; + public RoutingTableIncrementalDiff(StreamInput in) throws IOException { + version = in.readLong(); + indicesRouting = DiffableUtils.readJdkMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER + ); } - /** - * Reads a {@link RoutingTableIncrementalDiff} from the given {@link StreamInput}. - * - * @param in the input stream to read from. - * @return the deserialized RoutingTableIncrementalDiff. - * @throws IOException if an I/O exception occurs while reading from the stream. - */ public static RoutingTableIncrementalDiff readFrom(StreamInput in) throws IOException { - int size = in.readVInt(); - Map> diffs = new HashMap<>(); - - for (int i = 0; i < size; i++) { - String key = in.readString(); - Diff diff = IndexRoutingTableIncrementalDiff.readFrom(in); - diffs.put(key, diff); - } - return new RoutingTableIncrementalDiff(diffs); + return new RoutingTableIncrementalDiff(in); } - /** - * Applies the differences to the provided {@link RoutingTable}. - * - * @param part the original RoutingTable to which the differences will be applied. - * @return the updated RoutingTable with the applied differences. - */ @Override public RoutingTable apply(RoutingTable part) { - RoutingTable.Builder builder = new RoutingTable.Builder(); - for (IndexRoutingTable indexRoutingTable : part) { - builder.add(indexRoutingTable); // Add existing index routing tables to builder - } - - // Apply the diffs - for (Map.Entry> entry : diffs.entrySet()) { - builder.add(entry.getValue().apply(part.index(entry.getKey()))); - } - - return builder.build(); + return new RoutingTable(version, indicesRouting.apply(part.getIndicesRouting())); } - /** - * Writes the differences to the given {@link StreamOutput}. - * - * @param out the output stream to write to. - * @throws IOException if an I/O exception occurs while writing to the stream. - */ @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(diffs.size()); - for (Map.Entry> entry : diffs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); - } + out.writeLong(version); + indicesRouting.writeTo(out); + } + + @Override + public MapDiff> provideDiff() { + return (MapDiff>) indicesRouting; } /** @@ -103,66 +95,36 @@ public void writeTo(StreamOutput out) throws IOException { */ public static class IndexRoutingTableIncrementalDiff implements Diff { - private final List indexShardRoutingTables; + private final Diff> indexShardRoutingTables; - /** - * Constructs a new IndexShardRoutingTableDiff with the given shard routing tables. - * - * @param indexShardRoutingTables a list of IndexShardRoutingTable representing the differences. - */ - public IndexRoutingTableIncrementalDiff(List indexShardRoutingTables) { - this.indexShardRoutingTables = indexShardRoutingTables; + private final Index index; + + public IndexRoutingTableIncrementalDiff(Index index, IndexRoutingTable before, IndexRoutingTable after) { + this.index = index; + this.indexShardRoutingTables = DiffableUtils.diff(before.getShards(), after.getShards(), DiffableUtils.getIntKeySerializer()); + } + + private static final DiffableUtils.DiffableValueReader DIFF_VALUE_READER = + new DiffableUtils.DiffableValueReader<>(IndexShardRoutingTable::readFrom, IndexShardRoutingTable::readDiffFrom); + + public IndexRoutingTableIncrementalDiff(StreamInput in) throws IOException { + this.index = new Index(in); + this.indexShardRoutingTables = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getIntKeySerializer(), DIFF_VALUE_READER); } - /** - * Applies the differences to the provided {@link IndexRoutingTable}. - * - * @param part the original IndexRoutingTable to which the differences will be applied. - * @return the updated IndexRoutingTable with the applied differences. - */ @Override public IndexRoutingTable apply(IndexRoutingTable part) { - IndexRoutingTable.Builder builder = new IndexRoutingTable.Builder(part.getIndex()); - for (IndexShardRoutingTable shardRoutingTable : part) { - builder.addIndexShard(shardRoutingTable); // Add existing shards to builder - } - - // Apply the diff: update or add the new shard routing tables - for (IndexShardRoutingTable diffShard : indexShardRoutingTables) { - builder.addIndexShard(diffShard); - } - return builder.build(); + return new IndexRoutingTable(index, indexShardRoutingTables.apply(part.getShards())); } - /** - * Writes the differences to the given {@link StreamOutput}. - * - * @param out the output stream to write to. - * @throws IOException if an I/O exception occurs while writing to the stream. - */ @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(indexShardRoutingTables.size()); - for (IndexShardRoutingTable shardRoutingTable : indexShardRoutingTables) { - IndexShardRoutingTable.Builder.writeTo(shardRoutingTable, out); - } + index.writeTo(out); + indexShardRoutingTables.writeTo(out); } - /** - * Reads a {@link IndexRoutingTableIncrementalDiff} from the given {@link StreamInput}. - * - * @param in the input stream to read from. - * @return the deserialized IndexShardRoutingTableDiff. - * @throws IOException if an I/O exception occurs while reading from the stream. - */ public static IndexRoutingTableIncrementalDiff readFrom(StreamInput in) throws IOException { - int size = in.readVInt(); - List indexShardRoutingTables = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - IndexShardRoutingTable shardRoutingTable = IndexShardRoutingTable.Builder.readFrom(in); - indexShardRoutingTables.add(shardRoutingTable); - } - return new IndexRoutingTableIncrementalDiff(indexShardRoutingTables); + return new IndexRoutingTableIncrementalDiff(in); } } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/StringKeyDiffProvider.java b/server/src/main/java/org/opensearch/cluster/routing/StringKeyDiffProvider.java new file mode 100644 index 0000000000000..5d36a238e03ff --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/StringKeyDiffProvider.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.cluster.DiffableUtils; + +import java.util.Map; + +/** + * Interface for providing a difference (diff) between two maps with {@code String} keys and values of type {@code V}. + * This interface is used to compute and obtain the difference between two versions of a map, typically used + * in cluster state updates or other scenarios where changes need to be tracked and propagated efficiently. + * + * @param the type of the values in the map + */ +public interface StringKeyDiffProvider { + + /** + * Provides the difference between two versions of a map with {@code String} keys and values of type {@code V}. + * The difference is represented as a {@link DiffableUtils.MapDiff} object, which can be used to apply the + * changes to another map or to serialize the diff. + * + * @return a {@link DiffableUtils.MapDiff} object representing the difference between the maps + */ + DiffableUtils.MapDiff> provideDiff(); + +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index 0f1ff3138ef90..220093b428989 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -13,10 +13,10 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.Diff; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.remote.RemoteWritableEntityStore; @@ -63,7 +63,7 @@ public class InternalRemoteRoutingTableService extends AbstractLifecycleComponen private final Supplier repositoriesService; private Compressor compressor; private RemoteWritableEntityStore remoteIndexRoutingTableStore; - private RemoteWritableEntityStore remoteRoutingTableDiffStore; + private RemoteWritableEntityStore, RemoteRoutingTableDiff> remoteRoutingTableDiffStore; private final ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; @@ -95,16 +95,9 @@ public List getIndicesRouting(RoutingTable routingTable) { * @param after current routing table * @return incremental diff of the previous and current routing table */ - public DiffableUtils.MapDiff> getIndicesRoutingMapDiff( - RoutingTable before, - RoutingTable after - ) { - return DiffableUtils.diff( - before.getIndicesRouting(), - after.getIndicesRouting(), - DiffableUtils.getStringKeySerializer(), - CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER - ); + @Override + public StringKeyDiffProvider getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + return new RoutingTableIncrementalDiff(before, after); } /** @@ -142,18 +135,16 @@ public void getAsyncIndexRoutingDiffWriteAction( String clusterUUID, long term, long version, - Map> indexRoutingTableDiff, + StringKeyDiffProvider routingTableDiff, LatchedActionListener latchedActionListener ) { - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(indexRoutingTableDiff); RemoteRoutingTableDiff remoteRoutingTableDiff = new RemoteRoutingTableDiff( - routingTableIncrementalDiff, + (RoutingTableIncrementalDiff) routingTableDiff, clusterUUID, compressor, term, version ); - ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse(remoteRoutingTableDiff.getUploadedMetadata()), ex -> latchedActionListener.onFailure( @@ -209,15 +200,14 @@ public void getAsyncIndexRoutingReadAction( public void getAsyncIndexRoutingTableDiffReadAction( String clusterUUID, String uploadedFilename, - LatchedActionListener latchedActionListener + LatchedActionListener> latchedActionListener ) { - ActionListener actionListener = ActionListener.wrap( + ActionListener> actionListener = ActionListener.wrap( latchedActionListener::onResponse, latchedActionListener::onFailure ); RemoteRoutingTableDiff remoteRoutingTableDiff = new RemoteRoutingTableDiff(uploadedFilename, clusterUUID, compressor); - remoteRoutingTableDiffStore.readAsync(remoteRoutingTableDiff, actionListener); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java index 1ebf3206212a1..17687199c39d6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java @@ -10,16 +10,15 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.Diff; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.gateway.remote.ClusterMetadataManifest; import java.io.IOException; import java.util.List; -import java.util.Map; /** * Noop impl for RemoteRoutingTableService. @@ -32,16 +31,8 @@ public List getIndicesRouting(RoutingTable routingTable) { } @Override - public DiffableUtils.MapDiff> getIndicesRoutingMapDiff( - RoutingTable before, - RoutingTable after - ) { - return DiffableUtils.diff( - Map.of(), - Map.of(), - DiffableUtils.getStringKeySerializer(), - CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER - ); + public StringKeyDiffProvider getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + return new RoutingTableIncrementalDiff(RoutingTable.builder().build(), RoutingTable.builder().build()); } @Override @@ -60,7 +51,7 @@ public void getAsyncIndexRoutingDiffWriteAction( String clusterUUID, long term, long version, - Map> indexRoutingTableDiff, + StringKeyDiffProvider routingTableDiff, LatchedActionListener latchedActionListener ) { // noop @@ -89,7 +80,7 @@ public void getAsyncIndexRoutingReadAction( public void getAsyncIndexRoutingTableDiffReadAction( String clusterUUID, String uploadedFilename, - LatchedActionListener latchedActionListener + LatchedActionListener> latchedActionListener ) { // noop } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 0811a5f3010f4..d7ef3a29aa21f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,20 +10,14 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.Diff; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.common.lifecycle.LifecycleComponent; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.remote.ClusterMetadataManifest; import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * A Service which provides APIs to upload and download routing table from remote store. @@ -32,38 +26,6 @@ */ public interface RemoteRoutingTableService extends LifecycleComponent { - public static final DiffableUtils.DiffableValueSerializer CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER = - new DiffableUtils.DiffableValueSerializer() { - @Override - public IndexRoutingTable read(StreamInput in, String key) throws IOException { - return IndexRoutingTable.readFrom(in); - } - - @Override - public void write(IndexRoutingTable value, StreamOutput out) throws IOException { - value.writeTo(out); - } - - @Override - public Diff readDiff(StreamInput in, String key) throws IOException { - return IndexRoutingTable.readDiffFrom(in); - } - - @Override - public Diff diff(IndexRoutingTable currentState, IndexRoutingTable previousState) { - List diffs = new ArrayList<>(); - for (Map.Entry entry : currentState.getShards().entrySet()) { - Integer index = entry.getKey(); - IndexShardRoutingTable currentShardRoutingTable = entry.getValue(); - IndexShardRoutingTable previousShardRoutingTable = previousState.shard(index); - if (previousShardRoutingTable == null || !previousShardRoutingTable.equals(currentShardRoutingTable)) { - diffs.add(currentShardRoutingTable); - } - } - return new RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff(diffs); - } - }; - List getIndicesRouting(RoutingTable routingTable); void getAsyncIndexRoutingReadAction( @@ -75,7 +37,7 @@ void getAsyncIndexRoutingReadAction( void getAsyncIndexRoutingTableDiffReadAction( String clusterUUID, String uploadedFilename, - LatchedActionListener latchedActionListener + LatchedActionListener> latchedActionListener ); List getUpdatedIndexRoutingTableMetadata( @@ -83,10 +45,7 @@ List getUpdatedIndexRoutingTableM List allIndicesRouting ); - DiffableUtils.MapDiff> getIndicesRoutingMapDiff( - RoutingTable before, - RoutingTable after - ); + StringKeyDiffProvider getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after); void getAsyncIndexRoutingWriteAction( String clusterUUID, @@ -100,7 +59,7 @@ void getAsyncIndexRoutingDiffWriteAction( String clusterUUID, long term, long version, - Map> indexRoutingTableDiff, + StringKeyDiffProvider routingTableDiff, LatchedActionListener latchedActionListener ); @@ -110,8 +69,8 @@ List getAllUploadedIndicesRouting List indicesRoutingToDelete ); - public void deleteStaleIndexRoutingPaths(List stalePaths) throws IOException; + void deleteStaleIndexRoutingPaths(List stalePaths) throws IOException; - public void deleteStaleIndexRoutingDiffPaths(List stalePaths) throws IOException; + void deleteStaleIndexRoutingDiffPaths(List stalePaths) throws IOException; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index ca730a73dd89b..8025940b35997 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -45,9 +45,9 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { // also we introduce index routing-metadata, diff and other attributes as part of manifest // required for state publication public static final int CODEC_V3 = 3; // In Codec V3, we have introduced new diff field in diff-manifest's routing_table_diff + public static final int CODEC_V4 = 4; // In Codec V4, we have removed upserts and delete field for routing table in diff manifest - public static final int[] CODEC_VERSIONS = { CODEC_V0, CODEC_V1, CODEC_V2, CODEC_V3 }; - + public static final int[] CODEC_VERSIONS = { CODEC_V0, CODEC_V1, CODEC_V2, CODEC_V3, CODEC_V4 }; private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); private static final ParseField CLUSTER_UUID_FIELD = new ParseField("cluster_uuid"); @@ -116,6 +116,10 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields return manifestV2Builder(fields); } + private static ClusterMetadataManifest.Builder manifestV4Builder(Object[] fields) { + return manifestV3Builder(fields); + } + private static long term(Object[] fields) { return (long) fields[0]; } @@ -238,8 +242,13 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { fields -> manifestV3Builder(fields).build() ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; - public static final int MANIFEST_CURRENT_CODEC_VERSION = CODEC_V3; + private static final ConstructingObjectParser PARSER_V4 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> manifestV4Builder(fields).build() + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V4; + public static final int MANIFEST_CURRENT_CODEC_VERSION = CODEC_V4; private static final Map VERSION_TO_CODEC_MAPPING; @@ -248,6 +257,7 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); declareParser(PARSER_V3, CODEC_V3); + declareParser(PARSER_V4, CODEC_V4); assert Arrays.stream(CODEC_VERSIONS).max().getAsInt() == MANIFEST_CURRENT_CODEC_VERSION; Map versionToCodecMapping = new HashMap<>(); @@ -258,8 +268,10 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V1); } else if (version.onOrAfter(Version.V_2_15_0) && version.before(Version.V_2_16_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V2); - } else if (version.onOrAfter(Version.V_2_16_0)) { + } else if (version.onOrAfter(Version.V_2_16_0) && version.before(Version.V_2_17_0)) { versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V3); + } else if (version.onOrAfter(Version.V_2_17_0)) { + versionToCodecMapping.put(version, ClusterMetadataManifest.CODEC_V4); } } VERSION_TO_CODEC_MAPPING = Collections.unmodifiableMap(versionToCodecMapping); @@ -860,6 +872,10 @@ public static ClusterMetadataManifest fromXContentV2(XContentParser parser) thro return PARSER_V2.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV3(XContentParser parser) throws IOException { + return PARSER_V3.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index a3b36ddcff1a7..a97b3e1bc6c4b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -33,6 +34,7 @@ import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V3; /** @@ -81,7 +83,8 @@ public class ClusterStateDiffManifest implements ToXContentFragment, Writeable { public ClusterStateDiffManifest( ClusterState state, ClusterState previousState, - DiffableUtils.MapDiff> routingTableIncrementalDiff, + int codecVersion, + StringKeyDiffProvider routingTableDiff, String indicesRoutingDiffPath ) { fromStateUUID = previousState.stateUUID(); @@ -109,13 +112,14 @@ public ClusterStateDiffManifest( customMetadataUpdated = new ArrayList<>(customDiff.getDiffs().keySet()); customMetadataUpdated.addAll(customDiff.getUpserts().keySet()); customMetadataDeleted = customDiff.getDeletes(); - indicesRoutingUpdated = new ArrayList<>(); indicesRoutingDeleted = new ArrayList<>(); this.indicesRoutingDiffPath = indicesRoutingDiffPath; - if (routingTableIncrementalDiff != null) { - routingTableIncrementalDiff.getUpserts().forEach((k, v) -> indicesRoutingUpdated.add(k)); - indicesRoutingDeleted.addAll(routingTableIncrementalDiff.getDeletes()); + if (codecVersion == CODEC_V2 || codecVersion == CODEC_V3) { + if (routingTableDiff != null && routingTableDiff.provideDiff() != null) { + routingTableDiff.provideDiff().getUpserts().forEach((k, v) -> indicesRoutingUpdated.add(k)); + indicesRoutingDeleted.addAll(routingTableDiff.provideDiff().getDeletes()); + } } hashesOfConsistentSettingsUpdated = !state.metadata() .hashesOfConsistentSettings() @@ -231,18 +235,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); - builder.startObject(ROUTING_TABLE_DIFF); - builder.startArray(UPSERTS_FIELD); - for (String index : indicesRoutingUpdated) { - builder.value(index); + if (!indicesRoutingUpdated.isEmpty()) { + builder.startArray(UPSERTS_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); } - builder.endArray(); - builder.startArray(DELETES_FIELD); - for (String index : indicesRoutingDeleted) { - builder.value(index); + if (!indicesRoutingDeleted.isEmpty()) { + builder.startArray(DELETES_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); } - builder.endArray(); if (indicesRoutingDiffPath != null) { builder.field(DIFF_FIELD, indicesRoutingDiffPath); } @@ -345,10 +352,14 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser, long parser.nextToken(); switch (currentFieldName) { case UPSERTS_FIELD: - builder.indicesRoutingUpdated(convertListToString(parser.listOrderedMap())); + if (codec_version == CODEC_V2 || codec_version == CODEC_V3) { + builder.indicesRoutingUpdated(convertListToString(parser.listOrderedMap())); + } break; case DELETES_FIELD: - builder.indicesRoutingDeleted(convertListToString(parser.listOrderedMap())); + if (codec_version == CODEC_V2 || codec_version == CODEC_V3) { + builder.indicesRoutingDeleted(convertListToString(parser.listOrderedMap())); + } break; case DIFF_FIELD: if (codec_version >= CODEC_V3) { @@ -466,14 +477,14 @@ public boolean isHashesOfConsistentSettingsUpdated() { return hashesOfConsistentSettingsUpdated; } - public List getIndicesRoutingUpdated() { - return indicesRoutingUpdated; - } - public String getIndicesRoutingDiffPath() { return indicesRoutingDiffPath; } + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + public List getIndicesRoutingDeleted() { return indicesRoutingDeleted; } @@ -666,6 +677,11 @@ public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { return this; } + public Builder indicesRoutingDiffPath(String indicesRoutingDiffPath) { + this.indicesRoutingDiff = indicesRoutingDiffPath; + return this; + } + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { this.indicesRoutingUpdated = indicesRoutingUpdated; return this; @@ -676,11 +692,6 @@ public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { return this; } - public Builder indicesRoutingDiffPath(String indicesRoutingDiffPath) { - this.indicesRoutingDiff = indicesRoutingDiffPath; - return this; - } - public Builder clusterStateCustomUpdated(List clusterStateCustomUpdated) { this.clusterStateCustomUpdated = clusterStateCustomUpdated; return this; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2b3913ced0144..098a36c7d42c4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -28,7 +28,7 @@ import org.opensearch.cluster.node.DiscoveryNodes.Builder; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory; import org.opensearch.cluster.service.ClusterService; @@ -93,6 +93,8 @@ import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V3; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; @@ -246,6 +248,7 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat ClusterStateDiffManifest clusterStateDiffManifest = new ClusterStateDiffManifest( clusterState, ClusterState.EMPTY_STATE, + MANIFEST_CURRENT_CODEC_VERSION, null, null ); @@ -345,12 +348,18 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( } final List indicesRoutingToUpload = new ArrayList<>(); - final DiffableUtils.MapDiff> routingTableIncrementalDiff = - remoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); - - Map> indexRoutingTableDiffs = routingTableIncrementalDiff.getDiffs(); - routingTableIncrementalDiff.getDiffs().forEach((k, v) -> indicesRoutingToUpload.add(clusterState.getRoutingTable().index(k))); - routingTableIncrementalDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); + final List deletedIndicesRouting = new ArrayList<>(); + final StringKeyDiffProvider routingTableDiff = remoteRoutingTableService.getIndicesRoutingMapDiff( + previousClusterState.getRoutingTable(), + clusterState.getRoutingTable() + ); + if (routingTableDiff != null && routingTableDiff.provideDiff() != null) { + routingTableDiff.provideDiff() + .getDiffs() + .forEach((k, v) -> indicesRoutingToUpload.add(clusterState.getRoutingTable().index(k))); + routingTableDiff.provideDiff().getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); + deletedIndicesRouting.addAll(routingTableDiff.provideDiff().getDeletes()); + } UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, @@ -387,7 +396,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( clusterStateCustomsDiff.getUpserts(), updateHashesOfConsistentSettings, indicesRoutingToUpload, - indexRoutingTableDiffs + routingTableDiff ); // update the map if the metadata was uploaded @@ -429,13 +438,14 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults.uploadedIndicesRoutingMetadata = remoteRoutingTableService.getAllUploadedIndicesRouting( previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, - routingTableIncrementalDiff.getDeletes() + deletedIndicesRouting ); ClusterStateDiffManifest clusterStateDiffManifest = new ClusterStateDiffManifest( clusterState, previousClusterState, - routingTableIncrementalDiff, + MANIFEST_CURRENT_CODEC_VERSION, + routingTableDiff, uploadedMetadataResults.uploadedIndicesRoutingDiffMetadata != null ? uploadedMetadataResults.uploadedIndicesRoutingDiffMetadata.getUploadedFilename() : null @@ -517,14 +527,18 @@ UploadedMetadataResults writeMetadataInParallel( Map clusterStateCustomToUpload, boolean uploadHashesOfConsistentSettings, List indicesRoutingToUpload, - Map> indexRoutingTableDiff + StringKeyDiffProvider routingTableDiff ) throws IOException { assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null"; int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + (uploadTransientSettingMetadata ? 1 : 0) + clusterStateCustomToUpload.size() + (uploadHashesOfConsistentSettings ? 1 : 0) + indicesRoutingToUpload.size() - + (indexRoutingTableDiff != null && !indexRoutingTableDiff.isEmpty() ? 1 : 0); + + ((routingTableDiff != null + && routingTableDiff.provideDiff() != null + && (!routingTableDiff.provideDiff().getDiffs().isEmpty() + || !routingTableDiff.provideDiff().getDeletes().isEmpty() + || !routingTableDiff.provideDiff().getUpserts().isEmpty())) ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); List uploadTasks = Collections.synchronizedList(new ArrayList<>(totalUploadTasks)); Map results = new ConcurrentHashMap<>(totalUploadTasks); @@ -694,13 +708,17 @@ UploadedMetadataResults writeMetadataInParallel( listener ); }); - if (indexRoutingTableDiff != null && !indexRoutingTableDiff.isEmpty()) { + if (routingTableDiff != null + && routingTableDiff.provideDiff() != null + && (!routingTableDiff.provideDiff().getDiffs().isEmpty() + || !routingTableDiff.provideDiff().getDeletes().isEmpty() + || !routingTableDiff.provideDiff().getUpserts().isEmpty())) { uploadTasks.add(RemoteRoutingTableDiff.ROUTING_TABLE_DIFF_FILE); remoteRoutingTableService.getAsyncIndexRoutingDiffWriteAction( clusterState.metadata().clusterUUID(), clusterState.term(), clusterState.version(), - indexRoutingTableDiff, + routingTableDiff, listener ); } @@ -1024,7 +1042,7 @@ ClusterState readClusterStateInParallel( Map clusterStateCustomToRead, boolean readIndexRoutingTableDiff, boolean includeEphemeral - ) throws IOException { + ) { int totalReadTasks = indicesToRead.size() + customToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) @@ -1033,7 +1051,7 @@ ClusterState readClusterStateInParallel( CountDownLatch latch = new CountDownLatch(totalReadTasks); List readResults = Collections.synchronizedList(new ArrayList<>()); List readIndexRoutingTableResults = Collections.synchronizedList(new ArrayList<>()); - AtomicReference readIndexRoutingTableDiffResults = new AtomicReference<>(); + AtomicReference> readIndexRoutingTableDiffResults = new AtomicReference<>(); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); LatchedActionListener listener = new LatchedActionListener<>(ActionListener.wrap(response -> { @@ -1076,7 +1094,7 @@ ClusterState readClusterStateInParallel( ); } - LatchedActionListener routingTableDiffLatchedActionListener = new LatchedActionListener<>( + LatchedActionListener> routingTableDiffLatchedActionListener = new LatchedActionListener<>( ActionListener.wrap(response -> { logger.debug("Successfully read routing table diff component from remote"); readIndexRoutingTableDiffResults.set(response); @@ -1297,15 +1315,12 @@ ClusterState readClusterStateInParallel( readIndexRoutingTableResults.forEach( indexRoutingTable -> indicesRouting.put(indexRoutingTable.getIndex().getName(), indexRoutingTable) ); - RoutingTableIncrementalDiff routingTableDiff = readIndexRoutingTableDiffResults.get(); + Diff routingTableDiff = readIndexRoutingTableDiffResults.get(); + RoutingTable newRoutingTable = new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting); if (routingTableDiff != null) { - routingTableDiff.getDiffs().forEach((key, diff) -> { - IndexRoutingTable previousIndexRoutingTable = indicesRouting.get(key); - IndexRoutingTable updatedTable = diff.apply(previousIndexRoutingTable); - indicesRouting.put(key, updatedTable); - }); + newRoutingTable = routingTableDiff.apply(previousState.getRoutingTable()); } - clusterStateBuilder.routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting)); + clusterStateBuilder.routingTable(newRoutingTable); return clusterStateBuilder.build(); } @@ -1370,8 +1385,7 @@ public ClusterState getClusterStateForManifest( return clusterState; } - public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) - throws IOException { + public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) { assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); ClusterStateDiffManifest diff = manifest.getDiffManifest(); @@ -1398,9 +1412,11 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C } List updatedIndexRouting = new ArrayList<>(); - updatedIndexRouting.addAll( - remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), manifest.getIndicesRouting()) - ); + if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { + updatedIndexRouting.addAll( + remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), manifest.getIndicesRouting()) + ); + } ClusterState updatedClusterState = readClusterStateInParallel( previousState, @@ -1444,9 +1460,10 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C } HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); - - for (String indexName : diff.getIndicesRoutingDeleted()) { - indexRoutingTables.remove(indexName); + if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { + for (String indexName : diff.getIndicesRoutingDeleted()) { + indexRoutingTables.remove(indexName); + } } ClusterState clusterState = clusterStateBuilder.stateUUID(manifest.getStateUUID()) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index e890c30700eb6..9460e240e1706 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -52,6 +52,9 @@ public class RemoteClusterMetadataManifest extends AbstractClusterMetadataWritea public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV2); + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V3 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV3); + /** * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. */ @@ -151,6 +154,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(); if (codecVersion == ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V3) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V3; } else if (codecVersion == ClusterMetadataManifest.CODEC_V2) { return CLUSTER_METADATA_MANIFEST_FORMAT_V2; } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteRoutingTableDiff.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteRoutingTableDiff.java index 2370417dc14df..b3e0e9e5763b7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteRoutingTableDiff.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteRoutingTableDiff.java @@ -10,6 +10,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; import org.opensearch.common.io.Streams; import org.opensearch.common.remote.AbstractClusterMetadataWriteableBlobEntity; @@ -22,7 +23,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; -import java.util.Map; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -30,7 +30,8 @@ * Represents a incremental difference between {@link org.opensearch.cluster.routing.RoutingTable} objects that can be serialized and deserialized. * This class is responsible for writing and reading the differences between RoutingTables to and from an input/output stream. */ -public class RemoteRoutingTableDiff extends AbstractClusterMetadataWriteableBlobEntity { +public class RemoteRoutingTableDiff extends AbstractClusterMetadataWriteableBlobEntity> { + private final RoutingTableIncrementalDiff routingTableIncrementalDiff; private long term; @@ -71,18 +72,6 @@ public RemoteRoutingTableDiff( this.version = version; } - /** - * Constructs a new RemoteRoutingTableDiff with the given differences. - * - * @param routingTableIncrementalDiff a RoutingTableIncrementalDiff object containing the differences of {@link IndexRoutingTable}. - * @param clusterUUID the cluster UUID. - * @param compressor the compressor to be used. - */ - public RemoteRoutingTableDiff(RoutingTableIncrementalDiff routingTableIncrementalDiff, String clusterUUID, Compressor compressor) { - super(clusterUUID, compressor); - this.routingTableIncrementalDiff = routingTableIncrementalDiff; - } - /** * Constructs a new RemoteIndexRoutingTableDiff with the given blob name, cluster UUID, and compressor. * @@ -101,9 +90,8 @@ public RemoteRoutingTableDiff(String blobName, String clusterUUID, Compressor co * * @return a map containing the differences. */ - public Map> getDiffs() { - assert routingTableIncrementalDiff != null; - return routingTableIncrementalDiff.getDiffs(); + public Diff getDiffs() { + return routingTableIncrementalDiff; } @Override @@ -144,7 +132,7 @@ public InputStream serialize() throws IOException { } @Override - public RoutingTableIncrementalDiff deserialize(InputStream in) throws IOException { + public Diff deserialize(InputStream in) throws IOException { return REMOTE_ROUTING_TABLE_DIFF_FORMAT.deserialize(blobName, Streams.readFully(in)); } } diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableDiffTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableDiffTests.java new file mode 100644 index 0000000000000..d8c93523c34b9 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableDiffTests.java @@ -0,0 +1,325 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.OpenSearchAllocationTestCase; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; +import org.opensearch.common.settings.Settings; +import org.junit.Before; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class RoutingTableDiffTests extends OpenSearchAllocationTestCase { + + private static final String TEST_INDEX_1 = "test1"; + private static final String TEST_INDEX_2 = "test2"; + private static final String TEST_INDEX_3 = "test3"; + private int numberOfShards; + private int numberOfReplicas; + private int shardsPerIndex; + private int totalNumberOfShards; + private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private final AllocationService ALLOCATION_SERVICE = createAllocationService( + Settings.builder() + .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries + .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE) + .put( + ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING.getKey(), + Integer.MAX_VALUE + ) + .build() + ); + private ClusterState clusterState; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + this.numberOfShards = randomIntBetween(1, 5); + this.numberOfReplicas = randomIntBetween(1, 5); + this.shardsPerIndex = this.numberOfShards * (this.numberOfReplicas + 1); + this.totalNumberOfShards = this.shardsPerIndex * 2; + logger.info("Setup test with {} shards and {} replicas.", this.numberOfShards, this.numberOfReplicas); + RoutingTable emptyRoutingTable = new RoutingTable.Builder().build(); + Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build(); + + RoutingTable testRoutingTable = new RoutingTable.Builder().add( + new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_1).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_1)).build() + ) + .add( + new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_2).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_2)).build() + ) + .build(); + this.clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(testRoutingTable) + .build(); + } + + /** + * puts primary shard indexRoutings into initializing state + */ + private void initPrimaries() { + logger.info("adding {} nodes and performing rerouting", this.numberOfReplicas + 1); + DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); + for (int i = 0; i < this.numberOfReplicas + 1; i++) { + discoBuilder = discoBuilder.add(newNode("node" + i)); + } + this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build(); + ClusterState rerouteResult = ALLOCATION_SERVICE.reroute(clusterState, "reroute"); + assertThat(rerouteResult, not(equalTo(this.clusterState))); + this.clusterState = rerouteResult; + } + + private void startInitializingShards(String index) { + logger.info("start primary shards for index {}", index); + clusterState = startInitializingShardsAndReroute(ALLOCATION_SERVICE, clusterState, index); + } + + private IndexMetadata.Builder createIndexMetadata(String indexName) { + return new IndexMetadata.Builder(indexName).settings(DEFAULT_SETTINGS) + .numberOfReplicas(this.numberOfReplicas) + .numberOfShards(this.numberOfShards); + } + + public void testRoutingTableUpserts() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + int expectedUnassignedShardCount = this.totalNumberOfShards - 2 * this.numberOfShards; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(expectedUnassignedShardCount)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build(); + ClusterState oldClusterState = clusterState; + + // create index routing table for TEST_INDEX_3 + metadata = Metadata.builder(metadata).put(createIndexMetadata(TEST_INDEX_3)).build(); + RoutingTable testRoutingTable = new RoutingTable.Builder(clusterState.routingTable()).add( + new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_3).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_3)).build() + ).build(); + this.clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(testRoutingTable) + .build(); + this.totalNumberOfShards = this.shardsPerIndex * 3; + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(expectedUnassignedShardCount + this.shardsPerIndex) + ); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + + public void testRoutingTableDeletes() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + int expectedUnassignedShardCount = this.totalNumberOfShards - 2 * this.numberOfShards; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(expectedUnassignedShardCount)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build(); + ClusterState oldClusterState = clusterState; + + // delete index routing table for TEST_INDEX_1 + metadata = Metadata.builder(metadata).put(createIndexMetadata(TEST_INDEX_3)).build(); + RoutingTable testRoutingTable = new RoutingTable.Builder(clusterState.routingTable()).remove(TEST_INDEX_1).build(); + this.clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(testRoutingTable) + .build(); + this.totalNumberOfShards = this.shardsPerIndex; + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(expectedUnassignedShardCount - this.numberOfShards * this.numberOfReplicas) + ); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + + public void testRoutingTableUpsertsWithDiff() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + int expectedUnassignedShardCount = this.totalNumberOfShards - 2 * this.numberOfShards; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(expectedUnassignedShardCount)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build(); + ClusterState oldClusterState = clusterState; + + // create index routing table for TEST_INDEX_3 + metadata = Metadata.builder(metadata).put(createIndexMetadata(TEST_INDEX_3)).build(); + RoutingTable testRoutingTable = new RoutingTable.Builder(clusterState.routingTable()).add( + new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_3).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_3)).build() + ).build(); + this.clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(testRoutingTable) + .build(); + this.totalNumberOfShards = this.shardsPerIndex * 3; + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(expectedUnassignedShardCount + this.shardsPerIndex) + ); + initPrimaries(); + clusterState = startRandomInitializingShard(clusterState, ALLOCATION_SERVICE, TEST_INDEX_2); + // assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards + 1)); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + + public void testRoutingTableDiffWithReplicaAdded() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + int expectedUnassignedShardCount = this.totalNumberOfShards - 2 * this.numberOfShards; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(expectedUnassignedShardCount)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + ClusterState oldClusterState = clusterState; + + // update replica count for TEST_INDEX_1 + RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()) + .updateNumberOfReplicas(this.numberOfReplicas + 1, new String[] { TEST_INDEX_1 }) + .build(); + Metadata metadata = Metadata.builder(clusterState.metadata()) + .updateNumberOfReplicas(this.numberOfReplicas + 1, new String[] { TEST_INDEX_1 }) + .build(); + clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metadata(metadata).build(); + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(expectedUnassignedShardCount + this.numberOfShards) + ); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + + public void testRoutingTableDiffWithReplicaRemoved() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + int expectedUnassignedShardCount = this.totalNumberOfShards - 2 * this.numberOfShards; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(expectedUnassignedShardCount)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + ClusterState oldClusterState = clusterState; + + // update replica count for TEST_INDEX_1 + RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()) + .updateNumberOfReplicas(this.numberOfReplicas - 1, new String[] { TEST_INDEX_1 }) + .build(); + Metadata metadata = Metadata.builder(clusterState.metadata()) + .updateNumberOfReplicas(this.numberOfReplicas - 1, new String[] { TEST_INDEX_1 }) + .build(); + clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metadata(metadata).build(); + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(expectedUnassignedShardCount - this.numberOfShards) + ); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + + public void testRoutingTableDiffsWithStartedState() { + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), is(this.totalNumberOfShards)); + initPrimaries(); + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(this.totalNumberOfShards - 2 * this.numberOfShards) + ); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(2 * this.numberOfShards)); + + startInitializingShards(TEST_INDEX_1); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(this.numberOfShards)); + int initializingExpected = this.numberOfShards + this.numberOfShards * this.numberOfReplicas; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected)); + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(this.totalNumberOfShards - initializingExpected - this.numberOfShards) + ); + + startInitializingShards(TEST_INDEX_2); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(2 * this.numberOfShards)); + initializingExpected = 2 * this.numberOfShards * this.numberOfReplicas; + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected)); + assertThat( + clusterState.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).size(), + is(this.totalNumberOfShards - initializingExpected - 2 * this.numberOfShards) + ); + ClusterState oldClusterState = clusterState; + // start a random replica to change a single shard routing + clusterState = startRandomInitializingShard(clusterState, ALLOCATION_SERVICE); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.INITIALIZING).size(), is(initializingExpected - 1)); + assertThat(clusterState.routingTable().shardsWithState(ShardRoutingState.STARTED).size(), is(2 * this.numberOfShards + 1)); + Diff fullDiff = clusterState.routingTable().diff(oldClusterState.getRoutingTable()); + Diff incrementalDiff = clusterState.routingTable().incrementalDiff(oldClusterState.getRoutingTable()); + RoutingTable newRoutingTable = incrementalDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTable.version()); + assertEquals(indexRoutingTable, newRoutingTable.index(indexRoutingTable.getIndex())); + } + RoutingTable newRoutingTableWithFullDiff = fullDiff.apply(oldClusterState.getRoutingTable()); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + assertEquals(clusterState.routingTable().version(), newRoutingTableWithFullDiff.version()); + assertEquals(indexRoutingTable, newRoutingTableWithFullDiff.index(indexRoutingTable.getIndex())); + } + } + +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 74254f1a1987f..5061de9161ab4 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -13,14 +13,13 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.Diff; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; @@ -57,10 +56,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.function.Supplier; @@ -68,6 +65,7 @@ import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.ClusterMetadataManifestTests.randomUploadedIndexMetadataList; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; @@ -187,10 +185,9 @@ public void testGetIndicesRoutingMapDiff() { RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable); - assertEquals(0, diff.getUpserts().size()); - assertEquals(0, diff.getDeletes().size()); + StringKeyDiffProvider diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable); + assertEquals(0, diff.provideDiff().getUpserts().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); // Reversing order to check for equality without order. IndexRoutingTable indexRouting = routingTable.getIndicesRouting().get(indexName); @@ -201,8 +198,8 @@ public void testGetIndicesRoutingMapDiff() { RoutingTable routingTable2 = RoutingTable.builder().add(indexRoutingTableReversed).build(); diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(0, diff.getUpserts().size()); - assertEquals(0, diff.getDeletes().size()); + assertEquals(0, diff.provideDiff().getUpserts().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); } public void testGetChangedIndicesRouting() { @@ -220,7 +217,10 @@ public void testGetChangedIndicesRouting() { assertEquals( 0, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size() + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()) + .provideDiff() + .getUpserts() + .size() ); // Reversing order to check for equality without order. @@ -234,7 +234,10 @@ public void testGetChangedIndicesRouting() { .build(); assertEquals( 0, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size() + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()) + .provideDiff() + .getUpserts() + .size() ); } @@ -259,13 +262,12 @@ public void testGetIndicesRoutingMapDiffIndexAdded() { ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); RoutingTable routingTable2 = RoutingTable.builder(routingTable).addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(1, diff.getUpserts().size()); - assertNotNull(diff.getUpserts().get(indexName2)); - assertEquals(noOfShards, diff.getUpserts().get(indexName2).getShards().size()); + StringKeyDiffProvider diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.provideDiff().getUpserts().size()); + assertNotNull(diff.provideDiff().getUpserts().get(indexName2)); + assertEquals(noOfShards, diff.provideDiff().getUpserts().get(indexName2).getShards().size()); - assertEquals(0, diff.getDeletes().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); } public void testGetIndicesRoutingMapDiffShardChanged() { @@ -289,17 +291,19 @@ public void testGetIndicesRoutingMapDiffShardChanged() { ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(0, diff.getUpserts().size()); - assertEquals(1, diff.getDiffs().size()); - assertNotNull(diff.getDiffs().get(indexName)); - assertEquals(noOfShards + 1, diff.getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size()); + StringKeyDiffProvider diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(0, diff.provideDiff().getUpserts().size()); + assertEquals(1, diff.provideDiff().getDiffs().size()); + assertNotNull(diff.provideDiff().getDiffs().get(indexName)); + assertEquals( + noOfShards + 1, + diff.provideDiff().getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size() + ); assertEquals( noOfReplicas + 1, - diff.getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).getShards().get(0).getSize() + diff.provideDiff().getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).getShards().get(0).getSize() ); - assertEquals(0, diff.getDeletes().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); final IndexMetadata indexMetadata3 = new IndexMetadata.Builder(indexName).settings( Settings.builder() @@ -310,15 +314,18 @@ public void testGetIndicesRoutingMapDiffShardChanged() { RoutingTable routingTable3 = RoutingTable.builder().addAsNew(indexMetadata3).build(); diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable2, routingTable3); - assertEquals(0, diff.getUpserts().size()); - assertEquals(1, diff.getDiffs().size()); - assertNotNull(diff.getDiffs().get(indexName)); - assertEquals(noOfShards + 1, diff.getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size()); + assertEquals(0, diff.provideDiff().getUpserts().size()); + assertEquals(1, diff.provideDiff().getDiffs().size()); + assertNotNull(diff.provideDiff().getDiffs().get(indexName)); + assertEquals( + noOfShards + 1, + diff.provideDiff().getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size() + ); assertEquals( noOfReplicas + 2, - diff.getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).getShards().get(0).getSize() + diff.provideDiff().getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).getShards().get(0).getSize() ); - assertEquals(0, diff.getDeletes().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); } public void testGetIndicesRoutingMapDiffShardDetailChanged() { @@ -335,13 +342,15 @@ public void testGetIndicesRoutingMapDiffShardDetailChanged() { RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsRecovery(indexMetadata).build(); - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(1, diff.getDiffs().size()); - assertNotNull(diff.getDiffs().get(indexName)); - assertEquals(noOfShards, diff.getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size()); - assertEquals(0, diff.getUpserts().size()); - assertEquals(0, diff.getDeletes().size()); + StringKeyDiffProvider diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.provideDiff().getDiffs().size()); + assertNotNull(diff.provideDiff().getDiffs().get(indexName)); + assertEquals( + noOfShards, + diff.provideDiff().getDiffs().get(indexName).apply(routingTable.indicesRouting().get(indexName)).shards().size() + ); + assertEquals(0, diff.provideDiff().getUpserts().size()); + assertEquals(0, diff.provideDiff().getDeletes().size()); } public void testGetIndicesRoutingMapDiffIndexDeleted() { @@ -363,13 +372,12 @@ public void testGetIndicesRoutingMapDiffIndexDeleted() { ).numberOfShards(between(1, 1000)).numberOfReplicas(randomInt(10)).build(); RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(1, diff.getUpserts().size()); - assertNotNull(diff.getUpserts().get(indexName2)); + StringKeyDiffProvider diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable, routingTable2); + assertEquals(1, diff.provideDiff().getUpserts().size()); + assertNotNull(diff.provideDiff().getUpserts().get(indexName2)); - assertEquals(1, diff.getDeletes().size()); - assertEquals(indexName, diff.getDeletes().get(0)); + assertEquals(1, diff.provideDiff().getDeletes().size()); + assertEquals(indexName, diff.provideDiff().getDeletes().get(0)); } public void testGetAllUploadedIndicesRouting() { @@ -493,11 +501,17 @@ public void testIndicesRoutingDiffWhenIndexDeleted() { assertEquals( 1, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().size() + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() + .getDeletes() + .size() ); assertEquals( indexNameToDelete, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().get(0) + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() + .getDeletes() + .get(0) ); } @@ -524,19 +538,29 @@ public void testIndicesRoutingDiffWhenIndexDeletedAndAdded() { assertEquals( 1, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().size() + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() + .getDeletes() + .size() ); assertEquals( indexNameToDelete, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().get(0) + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() + .getDeletes() + .get(0) ); assertEquals( 1, - remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getUpserts().size() + remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() + .getUpserts() + .size() ); assertTrue( remoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable) + .provideDiff() .getUpserts() .containsKey(indexName) ); @@ -571,27 +595,17 @@ public void testGetAsyncIndexRoutingReadAction() throws Exception { public void testGetAsyncIndexRoutingTableDiffReadAction() throws Exception { String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - ClusterState currentState = createClusterState(indexName); - - // Get the IndexRoutingTable from the current state - IndexRoutingTable indexRoutingTable = currentState.routingTable().index(indexName); - Map shardRoutingTables = indexRoutingTable.getShards(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff indexRoutingTableDiff = - new RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff(new ArrayList<>(shardRoutingTables.values())); - - // Create the map for RoutingTableIncrementalDiff - Map> diffs = new HashMap<>(); - diffs.put(indexName, indexRoutingTableDiff); - - RoutingTableIncrementalDiff diff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff diff = new RoutingTableIncrementalDiff(previousState.getRoutingTable(), currentState.getRoutingTable()); String uploadedFileName = String.format(Locale.ROOT, "routing-table-diff/" + indexName); when(blobContainer.readBlob(indexName)).thenReturn( REMOTE_ROUTING_TABLE_DIFF_FORMAT.serialize(diff, uploadedFileName, compressor).streamInput() ); - TestCapturingListener listener = new TestCapturingListener<>(); + TestCapturingListener> listener = new TestCapturingListener<>(); CountDownLatch latch = new CountDownLatch(1); remoteRoutingTableService.getAsyncIndexRoutingTableDiffReadAction( @@ -603,8 +617,11 @@ public void testGetAsyncIndexRoutingTableDiffReadAction() throws Exception { assertNull(listener.getFailure()); assertNotNull(listener.getResult()); - RoutingTableIncrementalDiff resultDiff = listener.getResult(); - assertEquals(diff.getDiffs().size(), resultDiff.getDiffs().size()); + Diff resultDiff = listener.getResult(); + assertEquals( + currentState.getRoutingTable().getIndicesRouting(), + resultDiff.apply(previousState.getRoutingTable()).getIndicesRouting() + ); } public void testGetAsyncIndexRoutingWriteAction() throws Exception { @@ -661,20 +678,8 @@ public void testGetAsyncIndexRoutingWriteAction() throws Exception { public void testGetAsyncIndexRoutingDiffWriteAction() throws Exception { String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - ClusterState currentState = createClusterState(indexName); - - // Get the IndexRoutingTable from the current state - IndexRoutingTable indexRoutingTable = currentState.routingTable().index(indexName); - Map shardRoutingTables = indexRoutingTable.getShards(); - - RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff indexRoutingTableDiff = - new RoutingTableIncrementalDiff.IndexRoutingTableIncrementalDiff(new ArrayList<>(shardRoutingTables.values())); - - // Create the map for RoutingTableIncrementalDiff - Map> diffs = new HashMap<>(); - diffs.put(indexName, indexRoutingTableDiff); - - // RoutingTableIncrementalDiff diff = new RoutingTableIncrementalDiff(diffs); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); Iterable remotePath = new BlobPath().add("base-path") .add( @@ -699,7 +704,7 @@ public void testGetAsyncIndexRoutingDiffWriteAction() throws Exception { currentState.metadata().clusterUUID(), currentState.term(), currentState.version(), - diffs, + new RoutingTableIncrementalDiff(previousState.getRoutingTable(), currentState.getRoutingTable()), new LatchedActionListener<>(listener, latch) ); latch.await(); @@ -717,7 +722,7 @@ public void testGetAsyncIndexRoutingDiffWriteAction() throws Exception { assertEquals(4, fileNameTokens.length); assertEquals(ROUTING_TABLE_DIFF_METADATA_PREFIX, fileNameTokens[0]); assertEquals(RemoteStoreUtils.invertLong(1L), fileNameTokens[1]); - assertEquals(RemoteStoreUtils.invertLong(2L), fileNameTokens[2]); + assertEquals(RemoteStoreUtils.invertLong(1L), fileNameTokens[2]); assertThat(RemoteStoreUtils.invertLong(fileNameTokens[3]), lessThanOrEqualTo(System.currentTimeMillis())); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 418cfa104a911..6a191d002b971 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -10,11 +10,11 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,6 +40,7 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V3; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V4; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; @@ -199,6 +199,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { new ClusterStateDiffManifest( RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE, + ClusterMetadataManifest.CODEC_V3, null, "indicesRoutingDiffPath" ) @@ -533,6 +534,7 @@ public void testClusterMetadataManifestXContentV2() throws IOException { new ClusterStateDiffManifest( RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE, + CODEC_V2, null, null ) @@ -552,9 +554,7 @@ public void testClusterMetadataManifestXContentV2() throws IOException { public void testClusterMetadataManifestXContentV3() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - final DiffableUtils.MapDiff> routingTableIncrementalDiff = Mockito.mock( - DiffableUtils.MapDiff.class - ); + final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) @@ -599,6 +599,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { new ClusterStateDiffManifest( RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE, + CODEC_V3, routingTableIncrementalDiff, uploadedMetadataAttribute.getUploadedFilename() ) @@ -670,6 +671,71 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc } } + public void testClusterMetadataManifestXContentV4() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V4) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + CUSTOM_METADATA + CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .discoveryNodesMetadata(uploadedMetadataAttribute) + .clusterBlocksMetadata(uploadedMetadataAttribute) + .transientSettingsMetadata(uploadedMetadataAttribute) + .hashesOfConsistentSettings(uploadedMetadataAttribute) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest( + new ClusterStateDiffManifest( + RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), + ClusterState.EMPTY_STATE, + CODEC_V4, + routingTableIncrementalDiff, + uploadedMetadataAttribute.getUploadedFilename() + ) + ) + .build(); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + public static List randomUploadedIndexMetadataList() { final int size = randomIntBetween(1, 10); final List uploadedIndexMetadataList = new ArrayList<>(size); @@ -722,7 +788,7 @@ public void testGetCodecForVersion() { assertEquals(CODEC_V1, ClusterMetadataManifest.getCodecForVersion(Version.V_2_13_0)); assertEquals(CODEC_V2, ClusterMetadataManifest.getCodecForVersion(Version.V_2_15_0)); assertEquals(CODEC_V3, ClusterMetadataManifest.getCodecForVersion(Version.V_2_16_0)); - assertEquals(CODEC_V3, ClusterMetadataManifest.getCodecForVersion(Version.V_2_17_0)); + assertEquals(CODEC_V4, ClusterMetadataManifest.getCodecForVersion(Version.V_2_17_0)); } private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedIndexMetadata uploadedIndexMetadata) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index d57cc17acedcb..835087632d0d0 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -24,8 +24,8 @@ import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.cluster.routing.remote.NoopRemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; @@ -546,14 +546,14 @@ public void testTimeoutWhileWritingManifestFile() throws IOException { anyMap(), anyBoolean(), anyList(), - anyMap() + any() ) ).thenReturn(new RemoteClusterStateUtils.UploadedMetadataResults()); RemoteStateTransferException ex = expectThrows( RemoteStateTransferException.class, () -> spiedService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) ); - assertTrue(ex.getMessage().contains("Timed out waiting for transfer of following metadata to complete")); + assertTrue(ex.getMessage().contains("Timed out waiting for transfer")); } public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOException { @@ -697,7 +697,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { eq(Collections.emptyMap()), eq(false), eq(Collections.emptyList()), - eq(Collections.emptyMap()) + Mockito.any(StringKeyDiffProvider.class) ); assertThat(manifestInfo.getManifestFileName(), notNullValue()); @@ -778,7 +778,7 @@ public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws I eq(Collections.emptyMap()), eq(true), anyList(), - eq(Collections.emptyMap()) + Mockito.any(StringKeyDiffProvider.class) ); assertThat(manifestInfo.getManifestFileName(), notNullValue()); @@ -1256,11 +1256,6 @@ public void testGetClusterStateUsingDiff() throws IOException { ClusterState.Custom originalClusterStateCustom = clusterState.customs().get(clusterStateCustomName); assertNotEquals(originalClusterStateCustom, updateClusterStateCustom); }); - diffManifest.getIndicesRoutingUpdated().forEach(indexName -> { - IndexRoutingTable updatedIndexRoutingTable = updatedClusterState.getRoutingTable().getIndicesRouting().get(indexName); - IndexRoutingTable originalIndexingRoutingTable = clusterState.getRoutingTable().getIndicesRouting().get(indexName); - assertNotEquals(originalIndexingRoutingTable, updatedIndexRoutingTable); - }); diffManifest.getIndicesDeleted() .forEach(indexName -> { assertFalse(updatedClusterState.metadata().getIndices().containsKey(indexName)); }); diffManifest.getCustomMetadataDeleted().forEach(customMetadataName -> { @@ -1269,9 +1264,6 @@ public void testGetClusterStateUsingDiff() throws IOException { diffManifest.getClusterStateCustomDeleted().forEach(clusterStateCustomName -> { assertFalse(updatedClusterState.customs().containsKey(clusterStateCustomName)); }); - diffManifest.getIndicesRoutingDeleted().forEach(indexName -> { - assertFalse(updatedClusterState.getRoutingTable().getIndicesRouting().containsKey(indexName)); - }); } public void testReadClusterStateInParallel_TimedOut() throws IOException { @@ -2858,9 +2850,7 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTableDiffNull() throws INDEX_ROUTING_METADATA_PREFIX ); indices.add(uploadedIndiceRoutingMetadata); - final ClusterState previousClusterState = generateClusterStateWithOneIndex("test-index2", 5, 1, false).nodes( - nodesWithLocalNodeClusterManager() - ).build(); + final ClusterState previousClusterState = clusterState; final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(indices).build(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -2889,9 +2879,6 @@ public void testWriteIncrementalMetadataSuccessWithRoutingTableDiffNull() throws assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); - assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); - assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); - assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); assertThat(manifest.getDiffManifest().getIndicesRoutingDiffPath(), nullValue()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java index f89619a09cd52..e716822939a58 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java @@ -10,7 +10,6 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; @@ -19,6 +18,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; +import org.opensearch.cluster.routing.StringKeyDiffProvider; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; @@ -42,9 +43,9 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.Version.CURRENT; import static org.opensearch.cluster.ClusterState.EMPTY_STATE; -import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER; import static org.opensearch.core.common.transport.TransportAddress.META_ADDRESS; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V3; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V4; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks; @@ -120,7 +121,7 @@ public void testClusterStateDiffManifestXContent() throws IOException { diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterStateDiffManifest parsedManifest = ClusterStateDiffManifest.fromXContent(parser, CODEC_V3); + final ClusterStateDiffManifest parsedManifest = ClusterStateDiffManifest.fromXContent(parser, CODEC_V4); assertEquals(diffManifest, parsedManifest); } } @@ -132,7 +133,7 @@ public void testClusterStateWithRoutingTableDiffInDiffManifestXContent() throws ClusterState updatedState = generateClusterStateWithOneIndex("test-index", 5, 2, false).nodes(nodesWithLocalNodeClusterManager()) .build(); - ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState); + ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState, CODEC_V3); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -143,14 +144,14 @@ public void testClusterStateWithRoutingTableDiffInDiffManifestXContent() throws } } - public void testClusterStateWithRoutingTableDiffInDiffManifestXContent1() throws IOException { + public void testClusterStateWithRoutingTableDiffInDiffManifestXContentWithDeletes() throws IOException { ClusterState initialState = generateClusterStateWithOneIndex("test-index", 5, 1, true).nodes(nodesWithLocalNodeClusterManager()) .build(); ClusterState updatedState = generateClusterStateWithOneIndex("test-index-1", 5, 2, false).nodes(nodesWithLocalNodeClusterManager()) .build(); - ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState); + ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState, CODEC_V3); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -161,26 +162,60 @@ public void testClusterStateWithRoutingTableDiffInDiffManifestXContent1() throws } } - private ClusterStateDiffManifest verifyRoutingTableDiffManifest(ClusterState previousState, ClusterState currentState) { - // Create initial and updated IndexRoutingTable maps - Map initialRoutingTableMap = previousState.getRoutingTable().indicesRouting(); - Map updatedRoutingTableMap = currentState.getRoutingTable().indicesRouting(); + public void testClusterStateWithRoutingTableDiffInDiffManifestXContentV4() throws IOException { + ClusterState initialState = generateClusterStateWithOneIndex("test-index", 5, 1, true).nodes(nodesWithLocalNodeClusterManager()) + .build(); - DiffableUtils.MapDiff> routingTableIncrementalDiff = DiffableUtils.diff( - initialRoutingTableMap, - updatedRoutingTableMap, - DiffableUtils.getStringKeySerializer(), - CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER + ClusterState updatedState = generateClusterStateWithOneIndex("test-index", 5, 2, false).nodes(nodesWithLocalNodeClusterManager()) + .build(); + + ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState, CODEC_V4); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterStateDiffManifest parsedManifest = ClusterStateDiffManifest.fromXContent(parser, CODEC_V4); + assertEquals(diffManifest, parsedManifest); + } + } + + public void testClusterStateWithRoutingTableDiffInDiffManifestXContentWithDeletesV4() throws IOException { + ClusterState initialState = generateClusterStateWithOneIndex("test-index", 5, 1, true).nodes(nodesWithLocalNodeClusterManager()) + .build(); + + ClusterState updatedState = generateClusterStateWithOneIndex("test-index-1", 5, 2, false).nodes(nodesWithLocalNodeClusterManager()) + .build(); + + ClusterStateDiffManifest diffManifest = verifyRoutingTableDiffManifest(initialState, updatedState, CODEC_V4); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterStateDiffManifest parsedManifest = ClusterStateDiffManifest.fromXContent(parser, CODEC_V4); + assertEquals(diffManifest, parsedManifest); + } + } + + private ClusterStateDiffManifest verifyRoutingTableDiffManifest( + ClusterState previousState, + ClusterState currentState, + int codecVersion + ) { + // Create initial and updated IndexRoutingTable maps + StringKeyDiffProvider routingTableDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() ); ClusterStateDiffManifest manifest = new ClusterStateDiffManifest( currentState, previousState, - routingTableIncrementalDiff, + codecVersion, + routingTableDiff, "indicesRoutingDiffPath" ); assertEquals("indicesRoutingDiffPath", manifest.getIndicesRoutingDiffPath()); - assertEquals(routingTableIncrementalDiff.getUpserts().size(), manifest.getIndicesRoutingUpdated().size()); - assertEquals(routingTableIncrementalDiff.getDeletes().size(), manifest.getIndicesRoutingDeleted().size()); return manifest; } @@ -256,7 +291,7 @@ private ClusterStateDiffManifest updateAndVerifyState( } ClusterState updatedClusterState = clusterStateBuilder.metadata(metadataBuilder.build()).build(); - ClusterStateDiffManifest manifest = new ClusterStateDiffManifest(updatedClusterState, initialState, null, null); + ClusterStateDiffManifest manifest = new ClusterStateDiffManifest(updatedClusterState, initialState, CODEC_V4, null, null); assertEquals(indicesToAdd.stream().map(im -> im.getIndex().getName()).collect(toList()), manifest.getIndicesUpdated()); assertEquals(indicesToRemove, manifest.getIndicesDeleted()); assertEquals(new ArrayList<>(customsToAdd.keySet()), manifest.getCustomMetadataUpdated()); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiffTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiffTests.java index 6ffa7fc5cded8..5dcc3127f2f5d 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiffTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiffTests.java @@ -8,11 +8,8 @@ package org.opensearch.gateway.remote.routingtable; -import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.Diff; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.RoutingTableIncrementalDiff; import org.opensearch.common.blobstore.BlobPath; @@ -35,14 +32,12 @@ import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff.ROUTING_TABLE_DIFF_FILE; import static org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff.ROUTING_TABLE_DIFF_METADATA_PREFIX; import static org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff.ROUTING_TABLE_DIFF_PATH_TOKEN; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; @@ -87,22 +82,14 @@ public void tearDown() throws Exception { } public void testClusterUUID() { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); - - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); - - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -118,21 +105,14 @@ public void testClusterUUID() { } public void testFullBlobName() { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); - - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); - - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -148,21 +128,14 @@ public void testFullBlobName() { } public void testBlobFileName() { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); - - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); - - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -178,21 +151,14 @@ public void testBlobFileName() { } public void testBlobPathParameters() { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); - - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); - - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -210,21 +176,14 @@ public void testBlobPathParameters() { } public void testGenerateBlobFileName() { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); - - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); - - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -243,21 +202,14 @@ public void testGenerateBlobFileName() { } public void testGetUploadedMetadata() throws IOException { - Map> diffs = new HashMap<>(); String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int numberOfShards = randomIntBetween(1, 10); - int numberOfReplicas = randomIntBetween(1, 10); - - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - .build(); - - IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()).initializeAsNew(indexMetadata).build(); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, 5, 1, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, 5, 2, true).build(); - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( routingTableIncrementalDiff, @@ -277,41 +229,35 @@ public void testStreamOperations() throws IOException { int numberOfShards = randomIntBetween(1, 10); int numberOfReplicas = randomIntBetween(1, 10); - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder(indexName) - .settings(settings(Version.CURRENT)) - .numberOfShards(numberOfShards) - .numberOfReplicas(numberOfReplicas) - ) - .build(); - - RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build(); - Map> diffs = new HashMap<>(); - - initialRoutingTable.getIndicesRouting().values().forEach(indexRoutingTable -> { - diffs.put(indexName, indexRoutingTable.diff(indexRoutingTable)); - RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff(diffs); - - RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( - routingTableIncrementalDiff, - clusterUUID, - compressor, - STATE_TERM, - STATE_VERSION - ); - - assertThrows(AssertionError.class, remoteDiffForUpload::getUploadedMetadata); - - try (InputStream inputStream = remoteDiffForUpload.serialize()) { - remoteDiffForUpload.setFullBlobName(BlobPath.cleanPath()); - assertThat(inputStream.available(), greaterThan(0)); - - routingTableIncrementalDiff = remoteDiffForUpload.deserialize(inputStream); - assertEquals(remoteDiffForUpload.getDiffs().size(), routingTableIncrementalDiff.getDiffs().size()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + ClusterState previousState = generateClusterStateWithOneIndex(indexName, numberOfShards, numberOfReplicas, false).build(); + ClusterState currentState = generateClusterStateWithOneIndex(indexName, numberOfShards, numberOfReplicas + 1, true).build(); + + RoutingTableIncrementalDiff routingTableIncrementalDiff = new RoutingTableIncrementalDiff( + previousState.getRoutingTable(), + currentState.getRoutingTable() + ); + + RemoteRoutingTableDiff remoteDiffForUpload = new RemoteRoutingTableDiff( + routingTableIncrementalDiff, + clusterUUID, + compressor, + STATE_TERM, + STATE_VERSION + ); + + // Serialize the remote diff + InputStream inputStream = remoteDiffForUpload.serialize(); + + // Create a new instance for deserialization + RemoteRoutingTableDiff remoteDiffForDownload = new RemoteRoutingTableDiff(TEST_BLOB_NAME, clusterUUID, compressor); + + // Deserialize the remote diff + Diff deserializedDiff = remoteDiffForDownload.deserialize(inputStream); + + // Assert that the indices routing table created from routingTableIncrementalDiff and deserializedDiff is equal + assertEquals( + routingTableIncrementalDiff.apply(previousState.getRoutingTable()).getIndicesRouting(), + deserializedDiff.apply(previousState.getRoutingTable()).getIndicesRouting() + ); } } diff --git a/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationTestCase.java index f54ba36203684..bc42993ac2096 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/OpenSearchAllocationTestCase.java @@ -180,6 +180,19 @@ protected static DiscoveryNode newNode(String nodeId, Version version) { protected static ClusterState startRandomInitializingShard(ClusterState clusterState, AllocationService strategy) { List initializingShards = clusterState.getRoutingNodes().shardsWithState(INITIALIZING); + return startInitialisingShardsAndReroute(strategy, clusterState, initializingShards); + } + + protected static ClusterState startRandomInitializingShard(ClusterState clusterState, AllocationService strategy, String index) { + List initializingShards = clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING); + return startInitialisingShardsAndReroute(strategy, clusterState, initializingShards); + } + + private static ClusterState startInitialisingShardsAndReroute( + AllocationService strategy, + ClusterState clusterState, + List initializingShards + ) { if (initializingShards.isEmpty()) { return clusterState; } From 738cdd32287528ba62af662eaaa296b85dd0ffe1 Mon Sep 17 00:00:00 2001 From: Mohit Godwani <81609427+mgodwan@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:16:09 +0530 Subject: [PATCH 008/357] Update version check for context to comply with backport (#15572) Signed-off-by: Mohit Godwani --- .../java/org/opensearch/OpenSearchServerException.java | 3 ++- .../action/admin/indices/create/CreateIndexRequest.java | 4 ++-- .../action/admin/indices/get/GetIndexResponse.java | 4 ++-- .../org/opensearch/cluster/metadata/IndexMetadata.java | 8 ++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 7b9aded2cb740..b0ab62259d5c0 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -13,6 +13,7 @@ import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED; import static org.opensearch.Version.V_2_10_0; import static org.opensearch.Version.V_2_13_0; +import static org.opensearch.Version.V_2_17_0; import static org.opensearch.Version.V_2_1_0; import static org.opensearch.Version.V_2_4_0; import static org.opensearch.Version.V_2_5_0; @@ -1206,7 +1207,7 @@ public static void registerExceptions() { org.opensearch.indices.InvalidIndexContextException.class, org.opensearch.indices.InvalidIndexContextException::new, 174, - V_3_0_0 + V_2_17_0 ) ); registerExceptionHandle( diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java index 1634ebbad227b..6bb1bf0a3b97b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java @@ -132,7 +132,7 @@ public CreateIndexRequest(StreamInput in) throws IOException { aliases.add(new Alias(in)); } waitForActiveShards = ActiveShardCount.readFrom(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { context = in.readOptionalWriteable(Context::new); } } @@ -632,7 +632,7 @@ public void writeTo(StreamOutput out) throws IOException { alias.writeTo(out); } waitForActiveShards.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalWriteable(context); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java index cb4f466df40c8..a9ba991401bef 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/get/GetIndexResponse.java @@ -167,7 +167,7 @@ public GetIndexResponse( } dataStreams = Collections.unmodifiableMap(dataStreamsMapBuilder); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { final Map contextMapBuilder = new HashMap<>(); int contextSize = in.readVInt(); for (int i = 0; i < contextSize; i++) { @@ -297,7 +297,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(indexEntry.getValue()); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeVInt(contexts.size()); for (final Map.Entry indexEntry : contexts.entrySet()) { out.writeString(indexEntry.getKey()); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 618332deb0d15..405e9b3aabdb6 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1129,7 +1129,7 @@ private static class IndexMetadataDiff implements Diff { ); rolloverInfos = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), ROLLOVER_INFO_DIFF_VALUE_READER); isSystem = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { context = in.readOptionalWriteable(Context::new); } else { context = null; @@ -1153,7 +1153,7 @@ public void writeTo(StreamOutput out) throws IOException { inSyncAllocationIds.writeTo(out); rolloverInfos.writeTo(out); out.writeBoolean(isSystem); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalWriteable(context); } } @@ -1218,7 +1218,7 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { } builder.system(in.readBoolean()); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { builder.context(in.readOptionalWriteable(Context::new)); } return builder.build(); @@ -1259,7 +1259,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeBoolean(isSystem); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalWriteable(context); } } From 2e9db40a50735eacc95a4fc8926e8bb7042a696a Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 2 Sep 2024 17:59:01 -0400 Subject: [PATCH 009/357] Introduce ApproximateRangeQuery and ApproximateQuery (#13788) This introduces a basic "approximation" framework that improves query performance by modifying the query in a way that should be functionally equivalent. To start, we can reduce the bounds of a range query in order to satisfy the `track_total_hits` value (which defaults to 10,000). --------- Signed-off-by: Harsha Vamsi Kalluri Signed-off-by: Michael Froh Co-authored-by: Michael Froh --- CHANGELOG.md | 1 + .../test/search/370_approximate_range.yml | 72 +++ .../opensearch/common/util/FeatureFlags.java | 10 + .../index/mapper/DateFieldMapper.java | 32 +- .../bucket/filterrewrite/Helper.java | 2 + .../ApproximateIndexOrDocValuesQuery.java | 62 +++ .../ApproximatePointRangeQuery.java | 515 ++++++++++++++++++ .../search/approximate/ApproximateQuery.java | 21 + .../approximate/ApproximateScoreQuery.java | 90 +++ .../search/approximate/package-info.java | 12 + .../search/internal/ContextIndexSearcher.java | 4 + .../index/mapper/DateFieldTypeTests.java | 84 ++- ...angeFieldQueryStringQueryBuilderTests.java | 39 +- .../index/mapper/RangeFieldTypeTests.java | 11 +- .../query/MatchPhraseQueryBuilderTests.java | 2 + .../query/QueryStringQueryBuilderTests.java | 34 +- .../index/query/RangeQueryBuilderTests.java | 147 +++-- ...ApproximateIndexOrDocValuesQueryTests.java | 113 ++++ .../ApproximatePointRangeQueryTests.java | 346 ++++++++++++ .../ApproximateScoreQueryTests.java | 83 +++ 20 files changed, 1622 insertions(+), 58 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/370_approximate_range.yml create mode 100644 server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java create mode 100644 server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java create mode 100644 server/src/main/java/org/opensearch/search/approximate/ApproximateQuery.java create mode 100644 server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java create mode 100644 server/src/main/java/org/opensearch/search/approximate/package-info.java create mode 100644 server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java create mode 100644 server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java create mode 100644 server/src/test/java/org/opensearch/search/approximate/ApproximateScoreQueryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 67091eaa6b850..442677287daf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) - Add fieldType to AbstractQueryBuilder and FieldSortBuilder ([#15328](https://github.com/opensearch-project/OpenSearch/pull/15328))) - [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410)) +- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_approximate_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_approximate_range.yml new file mode 100644 index 0000000000000..ba896dfcad506 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_approximate_range.yml @@ -0,0 +1,72 @@ +--- +"search with approximate range": + - do: + indices.create: + index: test + body: + mappings: + properties: + date: + type: date + index: true + doc_values: true + + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {"_index": "test", "_id": "1" }}' + - '{ "date": "2018-10-29T12:12:12.987Z" }' + - '{ "index": { "_index": "test", "_id": "2" }}' + - '{ "date": "2020-10-29T12:12:12.987Z" }' + - '{ "index": { "_index": "test", "_id": "3" } }' + - '{ "date": "2024-10-29T12:12:12.987Z" }' + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z" + }, + } + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + sort: [{ date: asc }] + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z" + }, + } + + + - match: { hits.total: 3 } + - match: { hits.hits.0._id: "1" } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + sort: [{ date: desc }] + query: + range: { + date: { + gte: "2018-10-29T12:12:12.987Z", + lte: "2020-10-29T12:12:12.987Z" + }, + } + + - match: { hits.total: 2 } + - match: { hits.hits.0._id: "2" } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 14920da134e42..0fd5edde2b94c 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -130,6 +130,16 @@ public class FeatureFlags { Property.NodeScope ); + /** + * Gates the functionality of ApproximatePointRangeQuery where we approximate query results. + */ + public static final String APPROXIMATE_POINT_RANGE_QUERY = "opensearch.experimental.feature.approximate_point_range_query.enabled"; + public static final Setting APPROXIMATE_POINT_RANGE_QUERY_SETTING = Setting.boolSetting( + APPROXIMATE_POINT_RANGE_QUERY, + false, + Property.NodeScope + ); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index cf8703209fb37..e60d5a739a9ae 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -61,6 +61,8 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.lookup.SearchLookup; import java.io.IOException; @@ -80,6 +82,7 @@ import java.util.function.Supplier; import static org.opensearch.common.time.DateUtils.toLong; +import static org.apache.lucene.document.LongPoint.pack; /** * A {@link FieldMapper} for dates. @@ -108,6 +111,21 @@ public static DateFormatter getDefaultDateTimeFormatter() { : LEGACY_DEFAULT_DATE_TIME_FORMATTER; } + public static Query getDefaultQuery(Query pointRangeQuery, Query dvQuery, String name, long l, long u) { + return FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY_SETTING) + ? new ApproximateIndexOrDocValuesQuery( + pointRangeQuery, + new ApproximatePointRangeQuery(name, pack(new long[] { l }).bytes, pack(new long[] { u }).bytes, new long[] { l }.length) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + dvQuery + ) + : new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + } + /** * Resolution of the date time * @@ -463,24 +481,22 @@ public Query rangeQuery( } DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + Query pointRangeQuery = isSearchable() ? LongPoint.newRangeQuery(name(), l, u) : null; + Query dvQuery = hasDocValues() ? SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u) : null; if (isSearchable() && hasDocValues()) { - Query query = LongPoint.newRangeQuery(name(), l, u); - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - + Query query = getDefaultQuery(pointRangeQuery, dvQuery, name(), l, u); if (context.indexSortedOnField(name())) { query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } return query; } if (hasDocValues()) { - Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, dvQuery); } - return query; + return dvQuery; } - return LongPoint.newRangeQuery(name(), l, u); + return pointRangeQuery; }); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java index 7493754d8efa2..17da7e5712be8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java @@ -23,6 +23,7 @@ import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.query.DateRangeIncludingNowQuery; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.internal.SearchContext; import java.io.IOException; @@ -54,6 +55,7 @@ private Helper() {} queryWrappers.put(FunctionScoreQuery.class, q -> ((FunctionScoreQuery) q).getSubQuery()); queryWrappers.put(DateRangeIncludingNowQuery.class, q -> ((DateRangeIncludingNowQuery) q).getQuery()); queryWrappers.put(IndexOrDocValuesQuery.class, q -> ((IndexOrDocValuesQuery) q).getIndexQuery()); + queryWrappers.put(ApproximateIndexOrDocValuesQuery.class, q -> ((ApproximateIndexOrDocValuesQuery) q).getOriginalQuery()); } /** diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java new file mode 100644 index 0000000000000..b99e0a0cbf808 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; + +/** + * A wrapper around {@link IndexOrDocValuesQuery} that can be used to run approximate queries. + * It delegates to either {@link ApproximateQuery} or {@link IndexOrDocValuesQuery} based on whether the query can be approximated or not. + * @see ApproximateQuery + */ +public final class ApproximateIndexOrDocValuesQuery extends ApproximateScoreQuery { + + private final ApproximateQuery approximateIndexQuery; + private final IndexOrDocValuesQuery indexOrDocValuesQuery; + + public ApproximateIndexOrDocValuesQuery(Query indexQuery, ApproximateQuery approximateIndexQuery, Query dvQuery) { + super(new IndexOrDocValuesQuery(indexQuery, dvQuery), approximateIndexQuery); + this.approximateIndexQuery = approximateIndexQuery; + this.indexOrDocValuesQuery = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } + + @Override + public String toString(String field) { + return "ApproximateIndexOrDocValuesQuery(indexQuery=" + + indexOrDocValuesQuery.getIndexQuery().toString(field) + + ", approximateIndexQuery=" + + approximateIndexQuery.toString(field) + + ", dvQuery=" + + indexOrDocValuesQuery.getRandomAccessQuery().toString(field) + + ")"; + } + + @Override + public void visit(QueryVisitor visitor) { + indexOrDocValuesQuery.visit(visitor); + } + + @Override + public boolean equals(Object obj) { + if (sameClassAs(obj) == false) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int h = classHash(); + h = 31 * h + indexOrDocValuesQuery.getIndexQuery().hashCode(); + h = 31 * h + indexOrDocValuesQuery.getRandomAccessQuery().hashCode(); + return h; + } +} diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java new file mode 100644 index 0000000000000..cee8bc43d7ffd --- /dev/null +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java @@ -0,0 +1,515 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.IntsRef; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortOrder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * An approximate-able version of {@link PointRangeQuery}. It creates an instance of {@link PointRangeQuery} but short-circuits the intersect logic + * after {@code size} is hit + */ +public abstract class ApproximatePointRangeQuery extends ApproximateQuery { + private int size; + + private SortOrder sortOrder; + + public final PointRangeQuery pointRangeQuery; + + protected ApproximatePointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, int numDims) { + this(field, lowerPoint, upperPoint, numDims, 10_000, null); + } + + protected ApproximatePointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, int numDims, int size) { + this(field, lowerPoint, upperPoint, numDims, size, null); + } + + protected ApproximatePointRangeQuery(String field, byte[] lowerPoint, byte[] upperPoint, int numDims, int size, SortOrder sortOrder) { + this.size = size; + this.sortOrder = sortOrder; + this.pointRangeQuery = new PointRangeQuery(field, lowerPoint, upperPoint, numDims) { + @Override + protected String toString(int dimension, byte[] value) { + return super.toString(field); + } + }; + } + + public int getSize() { + return this.size; + } + + public void setSize(int size) { + this.size = size; + } + + public SortOrder getSortOrder() { + return this.sortOrder; + } + + public void setSortOrder(SortOrder sortOrder) { + this.sortOrder = sortOrder; + } + + @Override + public void visit(QueryVisitor visitor) { + pointRangeQuery.visit(visitor); + } + + @Override + public final ConstantScoreWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + Weight pointRangeQueryWeight = pointRangeQuery.createWeight(searcher, scoreMode, boost); + + return new ConstantScoreWeight(this, boost) { + + private final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(pointRangeQuery.getBytesPerDim()); + + // we pull this from PointRangeQuery since it is final + private boolean matches(byte[] packedValue) { + for (int dim = 0; dim < pointRangeQuery.getNumDims(); dim++) { + int offset = dim * pointRangeQuery.getBytesPerDim(); + if (comparator.compare(packedValue, offset, pointRangeQuery.getLowerPoint(), offset) < 0) { + // Doc's value is too low, in this dimension + return false; + } + if (comparator.compare(packedValue, offset, pointRangeQuery.getUpperPoint(), offset) > 0) { + // Doc's value is too high, in this dimension + return false; + } + } + return true; + } + + // we pull this from PointRangeQuery since it is final + private PointValues.Relation relate(byte[] minPackedValue, byte[] maxPackedValue) { + + boolean crosses = false; + + for (int dim = 0; dim < pointRangeQuery.getNumDims(); dim++) { + int offset = dim * pointRangeQuery.getBytesPerDim(); + + if (comparator.compare(minPackedValue, offset, pointRangeQuery.getUpperPoint(), offset) > 0 + || comparator.compare(maxPackedValue, offset, pointRangeQuery.getLowerPoint(), offset) < 0) { + return PointValues.Relation.CELL_OUTSIDE_QUERY; + } + + crosses |= comparator.compare(minPackedValue, offset, pointRangeQuery.getLowerPoint(), offset) < 0 + || comparator.compare(maxPackedValue, offset, pointRangeQuery.getUpperPoint(), offset) > 0; + } + + if (crosses) { + return PointValues.Relation.CELL_CROSSES_QUERY; + } else { + return PointValues.Relation.CELL_INSIDE_QUERY; + } + } + + public PointValues.IntersectVisitor getIntersectVisitor(DocIdSetBuilder result, long[] docCount) { + return new PointValues.IntersectVisitor() { + + DocIdSetBuilder.BulkAdder adder; + + @Override + public void grow(int count) { + adder = result.grow(count); + } + + @Override + public void visit(int docID) { + // it is possible that size < 1024 and docCount < size but we will continue to count through all the 1024 docs + // and collect less, but it won't hurt performance + if (docCount[0] < size) { + adder.add(docID); + docCount[0]++; + } + } + + @Override + public void visit(DocIdSetIterator iterator) throws IOException { + adder.add(iterator); + } + + @Override + public void visit(IntsRef ref) { + for (int i = 0; i < ref.length; i++) { + adder.add(ref.ints[ref.offset + i]); + } + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (matches(packedValue)) { + visit(docID); + } + } + + @Override + public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException { + if (matches(packedValue)) { + adder.add(iterator); + } + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return relate(minPackedValue, maxPackedValue); + } + }; + } + + // we pull this from PointRangeQuery since it is final + private boolean checkValidPointValues(PointValues values) throws IOException { + if (values == null) { + // No docs in this segment/field indexed any points + return false; + } + + if (values.getNumIndexDimensions() != pointRangeQuery.getNumDims()) { + throw new IllegalArgumentException( + "field=\"" + + pointRangeQuery.getField() + + "\" was indexed with numIndexDimensions=" + + values.getNumIndexDimensions() + + " but this query has numDims=" + + pointRangeQuery.getNumDims() + ); + } + if (pointRangeQuery.getBytesPerDim() != values.getBytesPerDimension()) { + throw new IllegalArgumentException( + "field=\"" + + pointRangeQuery.getField() + + "\" was indexed with bytesPerDim=" + + values.getBytesPerDimension() + + " but this query has bytesPerDim=" + + pointRangeQuery.getBytesPerDim() + ); + } + return true; + } + + private void intersectLeft(PointValues.PointTree pointTree, PointValues.IntersectVisitor visitor, long[] docCount) + throws IOException { + intersectLeft(visitor, pointTree, docCount); + assert pointTree.moveToParent() == false; + } + + private void intersectRight(PointValues.PointTree pointTree, PointValues.IntersectVisitor visitor, long[] docCount) + throws IOException { + intersectRight(visitor, pointTree, docCount); + assert pointTree.moveToParent() == false; + } + + // custom intersect visitor to walk the left of the tree + public void intersectLeft(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) + throws IOException { + PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); + if (docCount[0] > size) { + return; + } + switch (r) { + case CELL_OUTSIDE_QUERY: + // This cell is fully outside the query shape: stop recursing + break; + case CELL_INSIDE_QUERY: + // If the cell is fully inside, we keep moving to child until we reach a point where we can no longer move or when + // we have sufficient doc count. We first move down and then move to the left child + if (pointTree.moveToChild() && docCount[0] < size) { + do { + intersectLeft(visitor, pointTree, docCount); + } while (pointTree.moveToSibling() && docCount[0] < size); + pointTree.moveToParent(); + } else { + // we're at the leaf node, if we're under the size, visit all the docIds in this node. + if (docCount[0] < size) { + pointTree.visitDocIDs(visitor); + } + } + break; + case CELL_CROSSES_QUERY: + // The cell crosses the shape boundary, or the cell fully contains the query, so we fall + // through and do full filtering: + if (pointTree.moveToChild() && docCount[0] < size) { + do { + intersectLeft(visitor, pointTree, docCount); + } while (pointTree.moveToSibling() && docCount[0] < size); + pointTree.moveToParent(); + } else { + // TODO: we can assert that the first value here in fact matches what the pointTree + // claimed? + // Leaf node; scan and filter all points in this block: + if (docCount[0] < size) { + pointTree.visitDocValues(visitor); + } + } + break; + default: + throw new IllegalArgumentException("Unreachable code"); + } + } + + // custom intersect visitor to walk the right of tree + public void intersectRight(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) + throws IOException { + PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); + if (docCount[0] > size) { + return; + } + switch (r) { + case CELL_OUTSIDE_QUERY: + // This cell is fully outside the query shape: stop recursing + break; + + case CELL_INSIDE_QUERY: + // If the cell is fully inside, we keep moving right as long as the point tree size is over our size requirement + if (pointTree.size() > size && docCount[0] < size && moveRight(pointTree)) { + intersectRight(visitor, pointTree, docCount); + pointTree.moveToParent(); + } + // if point tree size is no longer over, we have to go back one level where it still was over and the intersect left + else if (pointTree.size() <= size && docCount[0] < size) { + pointTree.moveToParent(); + intersectLeft(visitor, pointTree, docCount); + } + // if we've reached leaf, it means out size is under the size of the leaf, we can just collect all docIDs + else { + // Leaf node; scan and filter all points in this block: + if (docCount[0] < size) { + pointTree.visitDocIDs(visitor); + } + } + break; + case CELL_CROSSES_QUERY: + // If the cell is fully inside, we keep moving right as long as the point tree size is over our size requirement + if (pointTree.size() > size && docCount[0] < size && moveRight(pointTree)) { + intersectRight(visitor, pointTree, docCount); + pointTree.moveToParent(); + } + // if point tree size is no longer over, we have to go back one level where it still was over and the intersect left + else if (pointTree.size() <= size && docCount[0] < size) { + pointTree.moveToParent(); + intersectLeft(visitor, pointTree, docCount); + } + // if we've reached leaf, it means out size is under the size of the leaf, we can just collect all doc values + else { + // Leaf node; scan and filter all points in this block: + if (docCount[0] < size) { + pointTree.visitDocValues(visitor); + } + } + break; + default: + throw new IllegalArgumentException("Unreachable code"); + } + } + + public boolean moveRight(PointValues.PointTree pointTree) throws IOException { + return pointTree.moveToChild() && pointTree.moveToSibling(); + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + LeafReader reader = context.reader(); + long[] docCount = { 0 }; + + PointValues values = reader.getPointValues(pointRangeQuery.getField()); + if (checkValidPointValues(values) == false) { + return null; + } + final Weight weight = this; + if (size > values.size()) { + return pointRangeQueryWeight.scorerSupplier(context); + } else { + if (sortOrder == null || sortOrder.equals(SortOrder.ASC)) { + return new ScorerSupplier() { + + final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, pointRangeQuery.getField()); + final PointValues.IntersectVisitor visitor = getIntersectVisitor(result, docCount); + long cost = -1; + + @Override + public Scorer get(long leadCost) throws IOException { + intersectLeft(values.getPointTree(), visitor, docCount); + DocIdSetIterator iterator = result.build().iterator(); + return new ConstantScoreScorer(weight, score(), scoreMode, iterator); + } + + @Override + public long cost() { + if (cost == -1) { + // Computing the cost may be expensive, so only do it if necessary + cost = values.estimateDocCount(visitor); + assert cost >= 0; + } + return cost; + } + }; + } else { + // we need to fetch size + deleted docs since the collector will prune away deleted docs resulting in fewer results + // than expected + final int deletedDocs = reader.numDeletedDocs(); + size += deletedDocs; + return new ScorerSupplier() { + + final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, pointRangeQuery.getField()); + final PointValues.IntersectVisitor visitor = getIntersectVisitor(result, docCount); + long cost = -1; + + @Override + public Scorer get(long leadCost) throws IOException { + intersectRight(values.getPointTree(), visitor, docCount); + DocIdSetIterator iterator = result.build().iterator(); + return new ConstantScoreScorer(weight, score(), scoreMode, iterator); + } + + @Override + public long cost() { + if (cost == -1) { + // Computing the cost may be expensive, so only do it if necessary + cost = values.estimateDocCount(visitor); + assert cost >= 0; + } + return cost; + } + }; + } + } + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + + @Override + public int count(LeafReaderContext context) throws IOException { + return pointRangeQueryWeight.count(context); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } + }; + } + + @Override + public boolean canApproximate(SearchContext context) { + if (context == null) { + return false; + } + if (context.aggregations() != null) { + return false; + } + if (!(context.query() instanceof ApproximateIndexOrDocValuesQuery)) { + return false; + } + this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo())); + if (context.request() != null && context.request().source() != null) { + FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(context.request().source()); + if (primarySortField != null + && primarySortField.missing() == null + && primarySortField.getFieldName().equals(((RangeQueryBuilder) context.request().source().query()).fieldName())) { + if (primarySortField.order() == SortOrder.DESC) { + this.setSortOrder(SortOrder.DESC); + } + } + } + return true; + } + + @Override + public final int hashCode() { + return pointRangeQuery.hashCode(); + } + + @Override + public final boolean equals(Object o) { + return sameClassAs(o) && equalsTo(getClass().cast(o)); + } + + private boolean equalsTo(ApproximatePointRangeQuery other) { + return Objects.equals(pointRangeQuery.getField(), other.pointRangeQuery.getField()) + && pointRangeQuery.getNumDims() == other.pointRangeQuery.getNumDims() + && pointRangeQuery.getBytesPerDim() == other.pointRangeQuery.getBytesPerDim() + && Arrays.equals(pointRangeQuery.getLowerPoint(), other.pointRangeQuery.getLowerPoint()) + && Arrays.equals(pointRangeQuery.getUpperPoint(), other.pointRangeQuery.getUpperPoint()); + } + + @Override + public final String toString(String field) { + final StringBuilder sb = new StringBuilder(); + if (pointRangeQuery.getField().equals(field) == false) { + sb.append(pointRangeQuery.getField()); + sb.append(':'); + } + + // print ourselves as "range per dimension" + for (int i = 0; i < pointRangeQuery.getNumDims(); i++) { + if (i > 0) { + sb.append(','); + } + + int startOffset = pointRangeQuery.getBytesPerDim() * i; + + sb.append('['); + sb.append( + toString( + i, + ArrayUtil.copyOfSubArray(pointRangeQuery.getLowerPoint(), startOffset, startOffset + pointRangeQuery.getBytesPerDim()) + ) + ); + sb.append(" TO "); + sb.append( + toString( + i, + ArrayUtil.copyOfSubArray(pointRangeQuery.getUpperPoint(), startOffset, startOffset + pointRangeQuery.getBytesPerDim()) + ) + ); + sb.append(']'); + } + + return sb.toString(); + } + + /** + * Returns a string of a single value in a human-readable format for debugging. This is used by + * {@link #toString()}. + * + * @param dimension dimension of the particular value + * @param value single value, never null + * @return human readable value for debugging + */ + protected abstract String toString(int dimension, byte[] value); +} diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximateQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximateQuery.java new file mode 100644 index 0000000000000..0e6faf396b671 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximateQuery.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.search.Query; +import org.opensearch.search.internal.SearchContext; + +/** + * Abstract class that can be inherited by queries that can be approximated. Queries should implement {@link #canApproximate(SearchContext)} to specify conditions on when they can be approximated +*/ +public abstract class ApproximateQuery extends Query { + + protected abstract boolean canApproximate(SearchContext context); + +} diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java new file mode 100644 index 0000000000000..d1dd32b239f28 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Weight; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; + +/** + * Entry-point for the approximation framework. + * This class is heavily inspired by {@link org.apache.lucene.search.IndexOrDocValuesQuery}. It acts as a wrapper that consumer two queries, a regular query and an approximate version of the same. By default, it executes the regular query and returns {@link Weight#scorer} for the original query. At run-time, depending on certain constraints, we can re-write the {@code Weight} to use the approximate weight instead. + */ +public class ApproximateScoreQuery extends Query { + + private final Query originalQuery; + private final ApproximateQuery approximationQuery; + + protected Query resolvedQuery; + + public ApproximateScoreQuery(Query originalQuery, ApproximateQuery approximationQuery) { + this.originalQuery = originalQuery; + this.approximationQuery = approximationQuery; + } + + public Query getOriginalQuery() { + return originalQuery; + } + + public ApproximateQuery getApproximationQuery() { + return approximationQuery; + } + + @Override + public final Query rewrite(IndexSearcher indexSearcher) throws IOException { + if (resolvedQuery == null) { + throw new IllegalStateException("Cannot rewrite resolved query without setContext being called"); + } + return resolvedQuery.rewrite(indexSearcher); + } + + public void setContext(SearchContext context) { + if (resolvedQuery != null) { + throw new IllegalStateException("Query already resolved, duplicate call to setContext"); + } + resolvedQuery = approximationQuery.canApproximate(context) ? approximationQuery : originalQuery; + }; + + @Override + public String toString(String s) { + return "ApproximateScoreQuery(originalQuery=" + + originalQuery.toString() + + ", approximationQuery=" + + approximationQuery.toString() + + ")"; + } + + @Override + public void visit(QueryVisitor queryVisitor) { + QueryVisitor v = queryVisitor.getSubVisitor(BooleanClause.Occur.MUST, this); + originalQuery.visit(v); + approximationQuery.visit(v); + } + + @Override + public boolean equals(Object o) { + if (!sameClassAs(o)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int h = classHash(); + h = 31 * h + originalQuery.hashCode(); + h = 31 * h + approximationQuery.hashCode(); + return h; + } +} diff --git a/server/src/main/java/org/opensearch/search/approximate/package-info.java b/server/src/main/java/org/opensearch/search/approximate/package-info.java new file mode 100644 index 0000000000000..1a09183c7d9fa --- /dev/null +++ b/server/src/main/java/org/opensearch/search/approximate/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Approximation query framework to approximate commonly used queries + */ +package org.opensearch.search.approximate; diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index fa00ace378df1..f118e4106db83 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -69,6 +69,7 @@ import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchService; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.search.dfs.AggregatedDfs; import org.opensearch.search.profile.ContextualProfileBreakdown; import org.opensearch.search.profile.Timer; @@ -218,6 +219,9 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws profiler.pollLastElement(); } return new ProfileWeight(query, weight, profile); + } else if (query instanceof ApproximateScoreQuery) { + ((ApproximateScoreQuery) query).setContext(searchContext); + return super.createWeight(query, scoreMode, boost); } else { return super.createWeight(query, scoreMode, boost); } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index db5e1e419de93..259904fc143a1 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -41,7 +41,6 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; @@ -53,6 +52,7 @@ import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.DateMathParser; import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -65,12 +65,18 @@ import org.opensearch.index.query.DateRangeIncludingNowQuery; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.joda.time.DateTimeZone; import java.io.IOException; import java.time.ZoneOffset; import java.util.Collections; +import static org.hamcrest.CoreMatchers.is; +import static org.apache.lucene.document.LongPoint.pack; +import static org.junit.Assume.assumeThat; + public class DateFieldTypeTests extends FieldTypeTestCase { private static final long nowInMillis = 0; @@ -206,10 +212,26 @@ public void testTermQuery() { MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date)).toInstant().toEpochMilli(); - Query expected = new IndexOrDocValuesQuery( + Query expected = new ApproximateIndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant, instant + 999), + new ApproximatePointRangeQuery( + "field", + pack(new long[] { instant }).bytes, + pack(new long[] { instant + 999 }).bytes, + new long[] { instant }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999) ); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); assertEquals(expected, ft.termQuery(date, context)); MappedFieldType unsearchable = new DateFieldType( @@ -256,10 +278,26 @@ public void testRangeQuery() throws IOException { String date2 = "2016-04-28T11:33:52"; long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli(); long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999; - Query expected = new IndexOrDocValuesQuery( + Query expected = new ApproximateIndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), + new ApproximatePointRangeQuery( + "field", + pack(new long[] { instant1 }).bytes, + pack(new long[] { instant2 }).bytes, + new long[] { instant1 }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) ); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); assertEquals( expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new IndexSearcher(new MultiReader())) @@ -268,11 +306,27 @@ public void testRangeQuery() throws IOException { instant1 = nowInMillis; instant2 = instant1 + 100; expected = new DateRangeIncludingNowQuery( - new IndexOrDocValuesQuery( + new ApproximateIndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), + new ApproximatePointRangeQuery( + "field", + pack(new long[] { instant1 }).bytes, + pack(new long[] { instant2 }).bytes, + new long[] { instant1 }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) ) ); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); assertEquals(expected, ft.rangeQuery("now", instant2, true, true, null, null, null, context)); MappedFieldType unsearchable = new DateFieldType( @@ -329,13 +383,31 @@ public void testRangeQueryWithIndexSort() { long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli(); long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999; - Query pointQuery = LongPoint.newRangeQuery("field", instant1, instant2); Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); Query expected = new IndexSortSortedNumericDocValuesRangeQuery( "field", instant1, instant2, - new IndexOrDocValuesQuery(pointQuery, dvQuery) + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + new ApproximatePointRangeQuery( + "field", + pack(new long[] { instant1 }).bytes, + pack(new long[] { instant2 }).bytes, + new long[] { instant1 }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + dvQuery + ) + ); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) ); assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context)); } diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java index 9dea7e13ac45e..7a8ac829bdd97 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java @@ -47,15 +47,21 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.time.DateMathParser; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; import java.net.InetAddress; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.either; import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.apache.lucene.document.LongPoint.pack; +import static org.junit.Assume.assumeThat; public class RangeFieldQueryStringQueryBuilderTests extends AbstractQueryTestCase { @@ -173,18 +179,39 @@ public void testDateRangeQuery() throws Exception { DateFieldMapper.DateFieldType dateType = (DateFieldMapper.DateFieldType) context.fieldMapper(DATE_FIELD_NAME); parser = dateType.dateMathParser; Query queryOnDateField = new QueryStringQueryBuilder(DATE_FIELD_NAME + ":[2010-01-01 TO 2018-01-01]").toQuery(createShardContext()); - Query controlQuery = LongPoint.newRangeQuery( - DATE_FIELD_NAME, - new long[] { parser.parse(lowerBoundExact, () -> 0).toEpochMilli() }, - new long[] { parser.parse(upperBoundExact, () -> 0).toEpochMilli() } - ); Query controlDv = SortedNumericDocValuesField.newSlowRangeQuery( DATE_FIELD_NAME, parser.parse(lowerBoundExact, () -> 0).toEpochMilli(), parser.parse(upperBoundExact, () -> 0).toEpochMilli() ); - assertEquals(new IndexOrDocValuesQuery(controlQuery, controlDv), queryOnDateField); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertEquals( + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery( + DATE_FIELD_NAME, + parser.parse(lowerBoundExact, () -> 0).toEpochMilli(), + parser.parse(upperBoundExact, () -> 0).toEpochMilli() + ), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { parser.parse(lowerBoundExact, () -> 0).toEpochMilli() }).bytes, + pack(new long[] { parser.parse(upperBoundExact, () -> 0).toEpochMilli() }).bytes, + new long[] { parser.parse(lowerBoundExact, () -> 0).toEpochMilli() }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + controlDv + ), + queryOnDateField + ); } public void testIPRangeQuery() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java index 49bf227e5073c..b157c43e45451 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java @@ -57,6 +57,7 @@ import org.opensearch.index.mapper.RangeFieldMapper.RangeFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.test.IndexSettingsModule; import org.joda.time.DateTime; import org.junit.Before; @@ -285,7 +286,15 @@ public void testDateRangeQueryUsingMappingFormatLegacy() { // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter); final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); - assertEquals("field:[1465975790000 TO 1466062190999]", ((IndexOrDocValuesQuery) queryOnDateField).getIndexQuery().toString()); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertEquals( + "field:[1465975790000 TO 1466062190999]", + ((IndexOrDocValuesQuery) ((ApproximateIndexOrDocValuesQuery) queryOnDateField).getOriginalQuery()).getIndexQuery().toString() + ); } public void testDateRangeQueryUsingMappingFormat() { diff --git a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java index f8d5d2ce3d062..ddf58073a5206 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java @@ -42,6 +42,7 @@ import org.apache.lucene.search.TermQuery; import org.opensearch.core.common.ParsingException; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -130,6 +131,7 @@ protected void doAssertLuceneQuery(MatchPhraseQueryBuilder queryBuilder, Query q .or(instanceOf(PointRangeQuery.class)) .or(instanceOf(IndexOrDocValuesQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)) + .or(instanceOf(ApproximateIndexOrDocValuesQuery.class)) ); } diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index af4a34aa98116..5b030df20e889 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -47,7 +47,6 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; -import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -71,11 +70,14 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.search.QueryStringQueryParser; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.test.AbstractQueryTestCase; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; @@ -98,6 +100,9 @@ import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.apache.lucene.document.LongPoint.pack; +import static org.junit.Assume.assumeThat; public class QueryStringQueryBuilderTests extends AbstractQueryTestCase { @@ -853,7 +858,12 @@ public void testToQueryDateWithTimeZone() throws Exception { QueryStringQueryBuilder qsq = queryStringQuery(DATE_FIELD_NAME + ":1970-01-01"); QueryShardContext context = createShardContext(); Query query = qsq.toQuery(context); - assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertThat(query, instanceOf(ApproximateIndexOrDocValuesQuery.class)); long lower = 0; // 1970-01-01T00:00:00.999 UTC long upper = 86399999; // 1970-01-01T23:59:59.999 UTC assertEquals(calculateExpectedDateQuery(lower, upper), query); @@ -862,10 +872,22 @@ public void testToQueryDateWithTimeZone() throws Exception { assertEquals(calculateExpectedDateQuery(lower + msPerHour, upper + msPerHour), qsq.timeZone("-01:00").toQuery(context)); } - private IndexOrDocValuesQuery calculateExpectedDateQuery(long lower, long upper) { - Query query = LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper); - Query dv = SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper); - return new IndexOrDocValuesQuery(query, dv); + private ApproximateIndexOrDocValuesQuery calculateExpectedDateQuery(long lower, long upper) { + return new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { lower }).bytes, + pack(new long[] { upper }).bytes, + new long[] { lower }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ); } public void testFuzzyNumeric() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index e72be29b85b63..799d7c7b63462 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.Term; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -47,12 +48,16 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.lucene.BytesRefs; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.ParsingException; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MappedFieldType.Relation; import org.opensearch.index.mapper.MapperService; +import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximatePointRangeQuery; +import org.opensearch.search.approximate.ApproximateQuery; import org.opensearch.test.AbstractQueryTestCase; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; @@ -65,9 +70,12 @@ import java.util.Map; import static org.opensearch.index.query.QueryBuilders.rangeQuery; +import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.sameInstance; +import static org.apache.lucene.document.LongPoint.pack; +import static org.junit.Assume.assumeThat; public class RangeQueryBuilderTests extends AbstractQueryTestCase { @Override @@ -183,9 +191,16 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, assertThat(termRangeQuery.includesLower(), equalTo(queryBuilder.includeLower())); assertThat(termRangeQuery.includesUpper(), equalTo(queryBuilder.includeUpper())); } else if (expectedFieldName.equals(DATE_FIELD_NAME)) { - assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); - query = ((IndexOrDocValuesQuery) query).getIndexQuery(); - assertThat(query, instanceOf(PointRangeQuery.class)); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertThat(query, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + Query approximationQuery = ((ApproximateIndexOrDocValuesQuery) query).getApproximationQuery(); + assertThat(approximationQuery, instanceOf(ApproximateQuery.class)); + Query originalQuery = ((ApproximateIndexOrDocValuesQuery) query).getOriginalQuery(); + assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class)); MapperService mapperService = context.getMapperService(); MappedFieldType mappedFieldType = mapperService.fieldType(expectedFieldName); final Long fromInMillis; @@ -234,7 +249,24 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, maxLong--; } } - assertEquals(LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), query); + assertEquals( + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { minLong }).bytes, + pack(new long[] { maxLong }).bytes, + new long[] { minLong }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, minLong, maxLong) + ), + query + ); } else if (expectedFieldName.equals(INT_FIELD_NAME)) { assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); query = ((IndexOrDocValuesQuery) query).getIndexQuery(); @@ -299,15 +331,33 @@ public void testDateRangeQueryFormat() throws IOException { + " }\n" + "}"; Query parsedQuery = parseQuery(query).toQuery(createShardContext()); - assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); - parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); - assertThat(parsedQuery, instanceOf(PointRangeQuery.class)); - + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + Query approximationQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getApproximationQuery(); + assertThat(approximationQuery, instanceOf(ApproximateQuery.class)); + Query originalQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getOriginalQuery(); + assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class)); + long lower = DateTime.parse("2012-01-01T00:00:00.000+00").getMillis(); + long upper = DateTime.parse("2030-01-01T00:00:00.000+00").getMillis() - 1; assertEquals( - LongPoint.newRangeQuery( - DATE_FIELD_NAME, - DateTime.parse("2012-01-01T00:00:00.000+00").getMillis(), - DateTime.parse("2030-01-01T00:00:00.000+00").getMillis() - 1 + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { lower }).bytes, + pack(new long[] { upper }).bytes, + new long[] { lower }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) ), parsedQuery ); @@ -339,15 +389,33 @@ public void testDateRangeBoundaries() throws IOException { + " }\n" + "}\n"; Query parsedQuery = parseQuery(query).toQuery(createShardContext()); - assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); - parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); - assertThat(parsedQuery, instanceOf(PointRangeQuery.class)); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + + long lower = DateTime.parse("2014-11-01T00:00:00.000+00").getMillis(); + long upper = DateTime.parse("2014-12-08T23:59:59.999+00").getMillis(); assertEquals( - LongPoint.newRangeQuery( - DATE_FIELD_NAME, - DateTime.parse("2014-11-01T00:00:00.000+00").getMillis(), - DateTime.parse("2014-12-08T23:59:59.999+00").getMillis() - ), + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { lower }).bytes, + pack(new long[] { upper }).bytes, + new long[] { lower }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ) + + , parsedQuery ); @@ -362,15 +430,27 @@ public void testDateRangeBoundaries() throws IOException { + " }\n" + "}"; parsedQuery = parseQuery(query).toQuery(createShardContext()); - assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); - parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); - assertThat(parsedQuery, instanceOf(PointRangeQuery.class)); + assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + lower = DateTime.parse("2014-11-30T23:59:59.999+00").getMillis() + 1; + upper = DateTime.parse("2014-12-08T00:00:00.000+00").getMillis() - 1; assertEquals( - LongPoint.newRangeQuery( - DATE_FIELD_NAME, - DateTime.parse("2014-11-30T23:59:59.999+00").getMillis() + 1, - DateTime.parse("2014-12-08T00:00:00.000+00").getMillis() - 1 - ), + new ApproximateIndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximatePointRangeQuery( + DATE_FIELD_NAME, + pack(new long[] { lower }).bytes, + pack(new long[] { upper }).bytes, + new long[] { lower }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }, + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ) + + , parsedQuery ); } @@ -391,9 +471,14 @@ public void testDateRangeQueryTimezone() throws IOException { Query parsedQuery = parseQuery(query).toQuery(context); assertThat(parsedQuery, instanceOf(DateRangeIncludingNowQuery.class)); parsedQuery = ((DateRangeIncludingNowQuery) parsedQuery).getQuery(); - assertThat(parsedQuery, instanceOf(IndexOrDocValuesQuery.class)); - parsedQuery = ((IndexOrDocValuesQuery) parsedQuery).getIndexQuery(); - assertThat(parsedQuery, instanceOf(PointRangeQuery.class)); + assumeThat( + "Using Approximate Range Query as default", + FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), + is(true) + ); + assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + parsedQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getApproximationQuery(); + assertThat(parsedQuery, instanceOf(ApproximateQuery.class)); // TODO what else can we assert query = "{\n" diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java new file mode 100644 index 0000000000000..47f87c6abf629 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; + +import static org.apache.lucene.document.LongPoint.pack; + +public class ApproximateIndexOrDocValuesQueryTests extends OpenSearchTestCase { + private Directory dir; + private IndexWriter w; + private DirectoryReader reader; + private IndexSearcher searcher; + + @Before + public void initSearcher() throws IOException { + dir = newDirectory(); + w = new IndexWriter(dir, newIndexWriterConfig()); + } + + @After + public void closeAllTheReaders() throws IOException { + reader.close(); + w.close(); + dir.close(); + } + + public void testApproximateIndexOrDocValuesQueryWeight() throws IOException { + + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + Query indexQuery = LongPoint.newRangeQuery("test-index", l, u); + + ApproximateQuery approximateIndexQuery = new ApproximatePointRangeQuery( + "test-index", + pack(new long[] { l }).bytes, + pack(new long[] { u }).bytes, + new long[] { l }.length + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery("test-index", l, u); + + ApproximateIndexOrDocValuesQuery approximateIndexOrDocValuesQuery = new ApproximateIndexOrDocValuesQuery( + indexQuery, + approximateIndexQuery, + dvQuery + ); + + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + approximateIndexOrDocValuesQuery.resolvedQuery = indexQuery; + + Weight weight = approximateIndexOrDocValuesQuery.rewrite(searcher).createWeight(searcher, ScoreMode.COMPLETE, 1f); + + assertTrue(weight instanceof ConstantScoreWeight); + + ApproximateQuery approximateIndexQueryCanApproximate = new ApproximatePointRangeQuery( + "test-index", + pack(new long[] { l }).bytes, + pack(new long[] { u }).bytes, + new long[] { l }.length + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + + public boolean canApproximate(SearchContext context) { + return true; + } + }; + + ApproximateIndexOrDocValuesQuery approximateIndexOrDocValuesQueryCanApproximate = new ApproximateIndexOrDocValuesQuery( + indexQuery, + approximateIndexQueryCanApproximate, + dvQuery + ); + + approximateIndexOrDocValuesQueryCanApproximate.resolvedQuery = approximateIndexQueryCanApproximate; + + Weight approximateIndexOrDocValuesQueryCanApproximateWeight = approximateIndexOrDocValuesQueryCanApproximate.rewrite(searcher) + .createWeight(searcher, ScoreMode.COMPLETE, 1f); + + // we get ConstantScoreWeight since we're expecting to call ApproximatePointRangeQuery + assertTrue(approximateIndexOrDocValuesQueryCanApproximateWeight instanceof ConstantScoreWeight); + + } +} diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java new file mode 100644 index 0000000000000..dd683d28f00f7 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -0,0 +1,346 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.apache.lucene.document.LongPoint.pack; +import static org.mockito.Mockito.mock; + +public class ApproximatePointRangeQueryTests extends OpenSearchTestCase { + + protected static final String DATE_FIELD_NAME = "mapped_date"; + + public void testApproximateRangeEqualsActualRange() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + for (int i = 0; i < 100; i++) { + int numPoints = RandomNumbers.randomIntBetween(random(), 1, 10); + Document doc = new Document(); + for (int j = 0; j < numPoints; j++) { + for (int v = 0; v < dims; v++) { + scratch[v] = RandomNumbers.randomLongBetween(random(), 0, 100); + } + doc.add(new LongPoint("point", scratch)); + } + iw.addDocument(doc); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = RandomNumbers.randomLongBetween(random(), -100, 200); + long upper = lower + RandomNumbers.randomLongBetween(random(), 0, 100); + Query approximateQuery = new ApproximatePointRangeQuery("point", pack(lower).bytes, pack(upper).bytes, dims) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + Query query = LongPoint.newRangeQuery("point", lower, upper); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + TopDocs topDocs1 = searcher.search(query, 10); + assertEquals(topDocs.totalHits, topDocs1.totalHits); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testApproximateRangeWithDefaultSize() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + int numPoints = 1000; + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + for (int v = 0; v < dims; v++) { + scratch[v] = i; + } + doc.add(new LongPoint("point", scratch)); + iw.addDocument(doc); + if (i % 15 == 0) iw.flush(); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = 0; + long upper = 1000; + Query approximateQuery = new ApproximatePointRangeQuery("point", pack(lower).bytes, pack(upper).bytes, dims) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + assertEquals(topDocs.totalHits, new TotalHits(1000, TotalHits.Relation.EQUAL_TO)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testApproximateRangeWithSizeUnderDefault() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + int numPoints = 1000; + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + for (int v = 0; v < dims; v++) { + scratch[v] = i; + } + doc.add(new LongPoint("point", scratch)); + iw.addDocument(doc); + if (i % 15 == 0) iw.flush(); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = 0; + long upper = 45; + Query approximateQuery = new ApproximatePointRangeQuery("point", pack(lower).bytes, pack(upper).bytes, dims, 10) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testApproximateRangeWithSizeOverDefault() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + int numPoints = 15000; + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + for (int v = 0; v < dims; v++) { + scratch[v] = i; + } + doc.add(new LongPoint("point", scratch)); + iw.addDocument(doc); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = 0; + long upper = 12000; + Query approximateQuery = new ApproximatePointRangeQuery( + "point", + pack(lower).bytes, + pack(upper).bytes, + dims, + 11_000 + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 11000); + assertEquals(topDocs.totalHits, new TotalHits(11001, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testApproximateRangeShortCircuit() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + int numPoints = 1000; + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + for (int v = 0; v < dims; v++) { + scratch[v] = i; + } + doc.add(new LongPoint("point", scratch)); + iw.addDocument(doc); + if (i % 10 == 0) iw.flush(); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = 0; + long upper = 100; + Query approximateQuery = new ApproximatePointRangeQuery("point", pack(lower).bytes, pack(upper).bytes, dims, 10) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + Query query = LongPoint.newRangeQuery("point", lower, upper); + ; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + TopDocs topDocs1 = searcher.search(query, 10); + + // since we short-circuit from the approx range at the end of size these will not be equal + assertNotEquals(topDocs.totalHits, topDocs1.totalHits); + assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); + assertEquals(topDocs1.totalHits, new TotalHits(101, TotalHits.Relation.EQUAL_TO)); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testApproximateRangeShortCircuitAscSort() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + int dims = 1; + + long[] scratch = new long[dims]; + int numPoints = 1000; + for (int i = 0; i < numPoints; i++) { + Document doc = new Document(); + for (int v = 0; v < dims; v++) { + scratch[v] = i; + } + doc.add(new LongPoint("point", scratch)); + iw.addDocument(doc); + } + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + long lower = 0; + long upper = 20; + Query approximateQuery = new ApproximatePointRangeQuery( + "point", + pack(lower).bytes, + pack(upper).bytes, + dims, + 10, + SortOrder.ASC + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + Query query = LongPoint.newRangeQuery("point", lower, upper); + ; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(approximateQuery, 10); + TopDocs topDocs1 = searcher.search(query, 10); + + // since we short-circuit from the approx range at the end of size these will not be equal + assertNotEquals(topDocs.totalHits, topDocs1.totalHits); + assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); + assertEquals(topDocs1.totalHits, new TotalHits(21, TotalHits.Relation.EQUAL_TO)); + assertEquals(topDocs.scoreDocs[0].doc, 0); + assertEquals(topDocs.scoreDocs[1].doc, 1); + assertEquals(topDocs.scoreDocs[2].doc, 2); + assertEquals(topDocs.scoreDocs[3].doc, 3); + assertEquals(topDocs.scoreDocs[4].doc, 4); + assertEquals(topDocs.scoreDocs[5].doc, 5); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } + + public void testSize() { + ApproximatePointRangeQuery query = new ApproximatePointRangeQuery("point", pack(0).bytes, pack(20).bytes, 1) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + assertEquals(query.getSize(), 10_000); + + query.setSize(100); + assertEquals(query.getSize(), 100); + + } + + public void testSortOrder() { + ApproximatePointRangeQuery query = new ApproximatePointRangeQuery("point", pack(0).bytes, pack(20).bytes, 1) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + assertNull(query.getSortOrder()); + + query.setSortOrder(SortOrder.ASC); + assertEquals(query.getSortOrder(), SortOrder.ASC); + } + + public void testCanApproximate() { + ApproximatePointRangeQuery query = new ApproximatePointRangeQuery("point", pack(0).bytes, pack(20).bytes, 1) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + + assertFalse(query.canApproximate(null)); + + ApproximatePointRangeQuery queryCanApproximate = new ApproximatePointRangeQuery("point", pack(0).bytes, pack(20).bytes, 1) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + + public boolean canApproximate(SearchContext context) { + return true; + } + }; + SearchContext searchContext = mock(SearchContext.class); + assertTrue(queryCanApproximate.canApproximate(searchContext)); + } +} diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximateScoreQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximateScoreQueryTests.java new file mode 100644 index 0000000000000..aa45ea6744227 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximateScoreQueryTests.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.approximate; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.apache.lucene.document.LongPoint.pack; + +public class ApproximateScoreQueryTests extends OpenSearchTestCase { + + public void testApproximationScoreSupplier() throws IOException { + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + Query originalQuery = new PointRangeQuery( + "test-index", + pack(new long[] { l }).bytes, + pack(new long[] { u }).bytes, + new long[] { l }.length + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + + ApproximateQuery approximateQuery = new ApproximatePointRangeQuery( + "test-index", + pack(new long[] { l }).bytes, + pack(new long[] { u }).bytes, + new long[] { l }.length + ) { + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + }; + + ApproximateScoreQuery query = new ApproximateScoreQuery(originalQuery, approximateQuery); + query.resolvedQuery = approximateQuery; + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { + Document document = new Document(); + document.add(new LongPoint("testPoint", Long.MIN_VALUE)); + iw.addDocument(document); + iw.flush(); + try (IndexReader reader = iw.getReader()) { + try { + IndexSearcher searcher = new IndexSearcher(reader); + searcher.search(query, 10); + Weight weight = query.rewrite(searcher).createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertEquals( + scorer, + originalQuery.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F).scorer(searcher.getLeafContexts().get(0)) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + } + } + } +} From cfcfe21263a39a577d30bcc9e223bb113025075a Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 3 Sep 2024 06:32:54 +0530 Subject: [PATCH 010/357] Change pinned timestamp file format stored in remote store (#15526) --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../RemoteStorePinnedTimestampsIT.java | 91 ++++++- .../remote/model/RemotePinnedTimestamps.java | 144 ---------- .../RemoteStorePinnedTimestampsBlobStore.java | 43 --- .../index/remote/RemoteStoreUtils.java | 7 +- .../RemoteStorePinnedTimestampService.java | 250 ++++++++---------- .../model/RemotePinnedTimestampsTests.java | 101 ------- ...toreDirectoryWithPinnedTimestampTests.java | 38 +-- ...oteFsTranslogWithPinnedTimestampTests.java | 67 ++--- 8 files changed, 233 insertions(+), 508 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemotePinnedTimestamps.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteStorePinnedTimestampsBlobStore.java delete mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemotePinnedTimestampsTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index cb91c63e17245..2fcda8c2d2f27 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.opensearch.action.LatchedActionListener; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -17,6 +18,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.util.Set; +import java.util.concurrent.CountDownLatch; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase { @@ -75,10 +77,25 @@ public void testTimestampPinUnpin() throws Exception { remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); - // This should be a no-op as pinning entity is different - remoteStorePinnedTimestampService.unpinTimestamp(timestamp1, "no-snapshot", noOpActionListener); // Unpinning already pinned entity remoteStorePinnedTimestampService.unpinTimestamp(timestamp2, "ss3", noOpActionListener); + + // This should fail as timestamp is not pinned by pinning entity + CountDownLatch latch = new CountDownLatch(1); + remoteStorePinnedTimestampService.unpinTimestamp(timestamp1, "no-snapshot", new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(Void unused) { + // onResponse should not get called. + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + }, latch)); + latch.await(); + // Adding different entity to already pinned timestamp remoteStorePinnedTimestampService.pinTimestamp(timestamp3, "ss5", noOpActionListener); @@ -93,4 +110,74 @@ public void testTimestampPinUnpin() throws Exception { remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); } + + public void testPinnedTimestampClone() throws Exception { + prepareCluster(1, 1, INDEX_NAME, 0, 2); + ensureGreen(INDEX_NAME); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + long timestamp1 = System.currentTimeMillis() + 30000L; + long timestamp2 = System.currentTimeMillis() + 60000L; + long timestamp3 = System.currentTimeMillis() + 900000L; + remoteStorePinnedTimestampService.pinTimestamp(timestamp1, "ss2", noOpActionListener); + remoteStorePinnedTimestampService.pinTimestamp(timestamp2, "ss3", noOpActionListener); + remoteStorePinnedTimestampService.pinTimestamp(timestamp3, "ss4", noOpActionListener); + + // Clone timestamp1 + remoteStorePinnedTimestampService.cloneTimestamp(timestamp1, "ss2", "ss2-2", noOpActionListener); + + // With clone, set of pinned timestamp will not change + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + assertBusy( + () -> assertEquals(Set.of(timestamp1, timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2()) + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + + // Clone timestamp1 but provide invalid existing entity + CountDownLatch latch = new CountDownLatch(1); + remoteStorePinnedTimestampService.cloneTimestamp( + timestamp1, + "ss3", + "ss2-3", + new LatchedActionListener<>(new ActionListener() { + @Override + public void onResponse(Void unused) { + // onResponse should not get called. + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + }, latch) + ); + latch.await(); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + assertBusy( + () -> assertEquals(Set.of(timestamp1, timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2()) + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + + // Now we have timestamp1 pinned by 2 entities, unpin 1, this should not change set of pinned timestamps + remoteStorePinnedTimestampService.unpinTimestamp(timestamp1, "ss2", noOpActionListener); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + assertBusy( + () -> assertEquals(Set.of(timestamp1, timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2()) + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + + // Now unpin second entity as well, set of pinned timestamp should be reduced by 1 + remoteStorePinnedTimestampService.unpinTimestamp(timestamp1, "ss2-2", noOpActionListener); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + assertBusy(() -> assertEquals(Set.of(timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2())); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePinnedTimestamps.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePinnedTimestamps.java deleted file mode 100644 index 030491cf8b7b9..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePinnedTimestamps.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.common.io.Streams; -import org.opensearch.common.remote.BlobPathParameters; -import org.opensearch.common.remote.RemoteWriteableBlobEntity; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.compress.Compressor; -import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumWritableBlobStoreFormat; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; - -/** - * Wrapper class for uploading/downloading {@link RemotePinnedTimestamps} to/from remote blob store - * - * @opensearch.internal - */ -public class RemotePinnedTimestamps extends RemoteWriteableBlobEntity { - private static final Logger logger = LogManager.getLogger(RemotePinnedTimestamps.class); - - /** - * Represents a collection of pinned timestamps and their associated pinning entities. - * This class is thread-safe and implements the Writeable interface for serialization. - */ - public static class PinnedTimestamps implements Writeable { - private final Map> pinnedTimestampPinningEntityMap; - - public PinnedTimestamps(Map> pinnedTimestampPinningEntityMap) { - this.pinnedTimestampPinningEntityMap = new ConcurrentHashMap<>(pinnedTimestampPinningEntityMap); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeMap(pinnedTimestampPinningEntityMap, StreamOutput::writeLong, StreamOutput::writeStringCollection); - } - - public static PinnedTimestamps readFrom(StreamInput in) throws IOException { - return new PinnedTimestamps(in.readMap(StreamInput::readLong, StreamInput::readStringList)); - } - - /** - * Pins a timestamp against a pinning entity. - * - * @param timestamp The timestamp to pin. - * @param pinningEntity The entity pinning the timestamp. - */ - public void pin(Long timestamp, String pinningEntity) { - logger.debug("Pinning timestamp = {} against entity = {}", timestamp, pinningEntity); - pinnedTimestampPinningEntityMap.computeIfAbsent(timestamp, k -> new ArrayList<>()).add(pinningEntity); - } - - /** - * Unpins a timestamp for a specific pinning entity. - * - * @param timestamp The timestamp to unpin. - * @param pinningEntity The entity unpinning the timestamp. - */ - public void unpin(Long timestamp, String pinningEntity) { - logger.debug("Unpinning timestamp = {} against entity = {}", timestamp, pinningEntity); - if (pinnedTimestampPinningEntityMap.containsKey(timestamp) == false - || pinnedTimestampPinningEntityMap.get(timestamp).contains(pinningEntity) == false) { - logger.warn("Timestamp: {} is not pinned by entity: {}", timestamp, pinningEntity); - } - pinnedTimestampPinningEntityMap.compute(timestamp, (k, v) -> { - v.remove(pinningEntity); - return v.isEmpty() ? null : v; - }); - } - - public Map> getPinnedTimestampPinningEntityMap() { - return new HashMap<>(pinnedTimestampPinningEntityMap); - } - } - - public static final String PINNED_TIMESTAMPS = "pinned_timestamps"; - public static final ChecksumWritableBlobStoreFormat PINNED_TIMESTAMPS_FORMAT = new ChecksumWritableBlobStoreFormat<>( - PINNED_TIMESTAMPS, - PinnedTimestamps::readFrom - ); - - private PinnedTimestamps pinnedTimestamps; - - public RemotePinnedTimestamps(String clusterUUID, Compressor compressor) { - super(clusterUUID, compressor); - pinnedTimestamps = new PinnedTimestamps(new HashMap<>()); - } - - @Override - public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of(PINNED_TIMESTAMPS), PINNED_TIMESTAMPS); - } - - @Override - public String getType() { - return PINNED_TIMESTAMPS; - } - - @Override - public String generateBlobFileName() { - return this.blobFileName = String.join(DELIMITER, PINNED_TIMESTAMPS, RemoteStoreUtils.invertLong(System.currentTimeMillis())); - } - - @Override - public InputStream serialize() throws IOException { - return PINNED_TIMESTAMPS_FORMAT.serialize(pinnedTimestamps, generateBlobFileName(), getCompressor()).streamInput(); - } - - @Override - public PinnedTimestamps deserialize(InputStream inputStream) throws IOException { - return PINNED_TIMESTAMPS_FORMAT.deserialize(blobName, Streams.readFully(inputStream)); - } - - public void setBlobFileName(String blobFileName) { - this.blobFileName = blobFileName; - } - - public void setPinnedTimestamps(PinnedTimestamps pinnedTimestamps) { - this.pinnedTimestamps = pinnedTimestamps; - } - - public PinnedTimestamps getPinnedTimestamps() { - return pinnedTimestamps; - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteStorePinnedTimestampsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteStorePinnedTimestampsBlobStore.java deleted file mode 100644 index 2a65dd993d0af..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteStorePinnedTimestampsBlobStore.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.remote.RemoteWriteableBlobEntity; -import org.opensearch.common.remote.RemoteWriteableEntityBlobStore; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * Extends the RemoteClusterStateBlobStore to support {@link RemotePinnedTimestamps} - */ -public class RemoteStorePinnedTimestampsBlobStore extends RemoteWriteableEntityBlobStore< - RemotePinnedTimestamps.PinnedTimestamps, - RemotePinnedTimestamps> { - - public static final String PINNED_TIMESTAMPS_PATH_TOKEN = "pinned_timestamps"; - private final BlobStoreRepository blobStoreRepository; - - public RemoteStorePinnedTimestampsBlobStore( - BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, - String clusterName, - ThreadPool threadPool, - String executor - ) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool, executor, PINNED_TIMESTAMPS_PATH_TOKEN); - this.blobStoreRepository = blobStoreRepository; - } - - @Override - public BlobPath getBlobPathForUpload(final RemoteWriteableBlobEntity obj) { - return blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN); - } -} diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index 871e2eb3ce47f..edcfe17d31fb8 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -542,8 +542,11 @@ public static List filterOutMetadataFilesBasedOnAge( if (RemoteStoreSettings.isPinnedTimestampsEnabled() == false) { return new ArrayList<>(metadataFiles); } - long maximumAllowedTimestamp = lastSuccessfulFetchOfPinnedTimestamps - RemoteStoreSettings.getPinnedTimestampsLookbackInterval() - .getMillis(); + // We allow now() - loopback interval to be pinned. Also, the actual pinning can take at most loopback interval + // This means the pinned timestamp can be available for read after at most (2 * loopback interval) + long maximumAllowedTimestamp = lastSuccessfulFetchOfPinnedTimestamps - (2 * RemoteStoreSettings + .getPinnedTimestampsLookbackInterval() + .getMillis()); List metadataFilesWithMinAge = new ArrayList<>(); for (String metadataFileName : metadataFiles) { long metadataTimestamp = getTimestampFunction.apply(metadataFileName); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index f7b262664d147..782ba5e9a6540 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -10,19 +10,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.ClusterName; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; -import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.model.RemotePinnedTimestamps; -import org.opensearch.gateway.remote.model.RemotePinnedTimestamps.PinnedTimestamps; -import org.opensearch.gateway.remote.model.RemoteStorePinnedTimestampsBlobStore; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; @@ -30,15 +26,13 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; +import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; -import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Set; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -51,17 +45,15 @@ public class RemoteStorePinnedTimestampService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteStorePinnedTimestampService.class); private static Tuple> pinnedTimestampsSet = new Tuple<>(-1L, Set.of()); - public static final int PINNED_TIMESTAMP_FILES_TO_KEEP = 5; + public static final String PINNED_TIMESTAMPS_PATH_TOKEN = "pinned_timestamps"; + public static final String PINNED_TIMESTAMPS_FILENAME_SEPARATOR = "__"; private final Supplier repositoriesService; private final Settings settings; private final ThreadPool threadPool; private final ClusterService clusterService; - private BlobStoreRepository blobStoreRepository; - private BlobStoreTransferService blobStoreTransferService; - private RemoteStorePinnedTimestampsBlobStore pinnedTimestampsBlobStore; + private BlobContainer blobContainer; private AsyncUpdatePinnedTimestampTask asyncUpdatePinnedTimestampTask; - private final Semaphore updateTimetampPinningSemaphore = new Semaphore(1); public RemoteStorePinnedTimestampService( Supplier repositoriesService, @@ -82,7 +74,6 @@ public RemoteStorePinnedTimestampService( */ public void start() { validateRemoteStoreConfiguration(); - initializeComponents(); startAsyncUpdateTask(RemoteStoreSettings.getPinnedTimestampsSchedulerInterval()); } @@ -93,19 +84,8 @@ private void validateRemoteStoreConfiguration() { assert remoteStoreRepo != null : "Remote Segment Store repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; - blobStoreRepository = (BlobStoreRepository) repository; - } - - private void initializeComponents() { - String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); - blobStoreTransferService = new BlobStoreTransferService(blobStoreRepository.blobStore(), this.threadPool); - pinnedTimestampsBlobStore = new RemoteStorePinnedTimestampsBlobStore( - blobStoreTransferService, - blobStoreRepository, - clusterName, - this.threadPool, - ThreadPool.Names.REMOTE_STATE_READ - ); + BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; + blobContainer = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN)); } private void startAsyncUpdateTask(TimeValue pinnedTimestampsSchedulerInterval) { @@ -129,101 +109,109 @@ public void pinTimestamp(long timestamp, String pinningEntity, ActionListener pinnedTimestamps.pin(timestamp, pinningEntity), listener); + long startTime = System.nanoTime(); + try { + logger.debug("Pinning timestamp = {} against entity = {}", timestamp, pinningEntity); + blobContainer.writeBlob(getBlobName(timestamp, pinningEntity), new ByteArrayInputStream(new byte[0]), 0, true); + long elapsedTime = System.nanoTime() - startTime; + if (elapsedTime > RemoteStoreSettings.getPinnedTimestampsLookbackInterval().nanos()) { + String errorMessage = String.format( + Locale.ROOT, + "Timestamp pinning took %s nanoseconds which is more than limit of %s nanoseconds, failing the operation", + elapsedTime, + RemoteStoreSettings.getPinnedTimestampsLookbackInterval().nanos() + ); + logger.error(errorMessage); + unpinTimestamp(timestamp, pinningEntity, ActionListener.wrap(() -> listener.onFailure(new RuntimeException(errorMessage)))); + } else { + listener.onResponse(null); + } + } catch (IOException e) { + listener.onFailure(e); + } } /** - * Unpins a timestamp from the remote store. + * Clones a timestamp by creating a new pinning entity for an existing timestamp. * - * @param timestamp The timestamp to be unpinned - * @param pinningEntity The entity responsible for unpinning the timestamp - * @param listener A listener to be notified when the unpinning operation completes + * This method attempts to create a new pinning entity for a given timestamp that is already + * associated with an existing pinning entity. If the timestamp exists for the existing entity, + * a new blob is created for the new pinning entity. If the timestamp doesn't exist for the + * existing entity, the operation fails with an IllegalArgumentException. + * + * @param timestamp The timestamp to be cloned. + * @param existingPinningEntity The name of the existing entity that has pinned the timestamp. + * @param newPinningEntity The name of the new entity to pin the timestamp to. + * @param listener An ActionListener that will be notified of the operation's success or failure. + * On success, onResponse will be called with null. On failure, onFailure will + * be called with the appropriate exception. */ - public void unpinTimestamp(long timestamp, String pinningEntity, ActionListener listener) { - updatePinning(pinnedTimestamps -> pinnedTimestamps.unpin(timestamp, pinningEntity), listener); - } - - private void updatePinning(Consumer updateConsumer, ActionListener listener) { - RemotePinnedTimestamps remotePinnedTimestamps = new RemotePinnedTimestamps( - clusterService.state().metadata().clusterUUID(), - blobStoreRepository.getCompressor() - ); - BlobPath path = pinnedTimestampsBlobStore.getBlobPathForUpload(remotePinnedTimestamps); + public void cloneTimestamp(long timestamp, String existingPinningEntity, String newPinningEntity, ActionListener listener) { try { - if (updateTimetampPinningSemaphore.tryAcquire(10, TimeUnit.MINUTES)) { - ActionListener semaphoreAwareListener = ActionListener.runBefore(listener, updateTimetampPinningSemaphore::release); - ActionListener> listCallResponseListener = getListenerForListCallResponse( - remotePinnedTimestamps, - updateConsumer, - semaphoreAwareListener - ); - blobStoreTransferService.listAllInSortedOrder( - path, - remotePinnedTimestamps.getType(), - Integer.MAX_VALUE, - listCallResponseListener - ); + logger.debug( + "cloning timestamp = {} with existing pinningEntity = {} with new pinningEntity = {}", + timestamp, + existingPinningEntity, + newPinningEntity + ); + String blobName = getBlobName(timestamp, existingPinningEntity); + if (blobContainer.blobExists(blobName)) { + logger.debug("Pinning timestamp = {} against entity = {}", timestamp, newPinningEntity); + blobContainer.writeBlob(getBlobName(timestamp, newPinningEntity), new ByteArrayInputStream(new byte[0]), 0, true); + listener.onResponse(null); } else { - throw new TimeoutException("Timed out while waiting to acquire lock in updatePinning"); + String errorMessage = String.format( + Locale.ROOT, + "Timestamp: %s is not pinned by existing entity: %s", + timestamp, + existingPinningEntity + ); + logger.error(errorMessage); + listener.onFailure(new IllegalArgumentException(errorMessage)); } - } catch (InterruptedException | TimeoutException e) { + } catch (IOException e) { listener.onFailure(e); } } - private ActionListener> getListenerForListCallResponse( - RemotePinnedTimestamps remotePinnedTimestamps, - Consumer updateConsumer, - ActionListener listener - ) { - return ActionListener.wrap(blobMetadata -> { - PinnedTimestamps pinnedTimestamps = new PinnedTimestamps(new HashMap<>()); - if (blobMetadata.isEmpty() == false) { - pinnedTimestamps = readExistingPinnedTimestamps(blobMetadata.get(0).name(), remotePinnedTimestamps); - } - updateConsumer.accept(pinnedTimestamps); - remotePinnedTimestamps.setPinnedTimestamps(pinnedTimestamps); - ActionListener writeCallResponseListener = getListenerForWriteCallResponse( - remotePinnedTimestamps, - blobMetadata, - listener - ); - pinnedTimestampsBlobStore.writeAsync(remotePinnedTimestamps, writeCallResponseListener); - }, listener::onFailure); + private String getBlobName(long timestamp, String pinningEntity) { + return String.join(PINNED_TIMESTAMPS_FILENAME_SEPARATOR, pinningEntity, String.valueOf(timestamp)); } - private ActionListener getListenerForWriteCallResponse( - RemotePinnedTimestamps remotePinnedTimestamps, - List blobMetadata, - ActionListener listener - ) { - return ActionListener.wrap(unused -> { - // Delete older pinnedTimestamp files - if (blobMetadata.size() > PINNED_TIMESTAMP_FILES_TO_KEEP) { - List oldFilesToBeDeleted = blobMetadata.subList(PINNED_TIMESTAMP_FILES_TO_KEEP, blobMetadata.size()) - .stream() - .map(BlobMetadata::name) - .collect(Collectors.toList()); - try { - blobStoreTransferService.deleteBlobs( - pinnedTimestampsBlobStore.getBlobPathForUpload(remotePinnedTimestamps), - oldFilesToBeDeleted - ); - } catch (IOException e) { - logger.error("Exception while deleting stale pinned timestamps", e); - } - } - listener.onResponse(null); - }, listener::onFailure); + private long getTimestampFromBlobName(String blobName) { + String[] blobNameTokens = blobName.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR); + if (blobNameTokens.length < 2) { + logger.error("Pinned timestamps blob name contains invalid format: {}", blobName); + } + try { + return Long.parseLong(blobNameTokens[blobNameTokens.length - 1]); + } catch (NumberFormatException e) { + logger.error(() -> new ParameterizedMessage("Pinned timestamps blob name contains invalid format: {}", blobName), e); + } + return -1; } - private PinnedTimestamps readExistingPinnedTimestamps(String blobFilename, RemotePinnedTimestamps remotePinnedTimestamps) { - remotePinnedTimestamps.setBlobFileName(blobFilename); - remotePinnedTimestamps.setFullBlobName(pinnedTimestampsBlobStore().getBlobPathForUpload(remotePinnedTimestamps)); + /** + * Unpins a timestamp from the remote store. + * + * @param timestamp The timestamp to be unpinned + * @param pinningEntity The entity responsible for unpinning the timestamp + * @param listener A listener to be notified when the unpinning operation completes + */ + public void unpinTimestamp(long timestamp, String pinningEntity, ActionListener listener) { try { - return pinnedTimestampsBlobStore().read(remotePinnedTimestamps); + logger.debug("Unpinning timestamp = {} against entity = {}", timestamp, pinningEntity); + String blobName = getBlobName(timestamp, pinningEntity); + if (blobContainer.blobExists(blobName)) { + blobContainer.deleteBlobsIgnoringIfNotExists(List.of(blobName)); + listener.onResponse(null); + } else { + String errorMessage = String.format(Locale.ROOT, "Timestamp: %s is not pinned by entity: %s", timestamp, pinningEntity); + logger.error(errorMessage); + listener.onFailure(new IllegalArgumentException(errorMessage)); + } } catch (IOException e) { - throw new RuntimeException("Failed to read existing pinned timestamps", e); + listener.onFailure(e); } } @@ -245,14 +233,6 @@ public static Tuple> getPinnedTimestamps() { return pinnedTimestampsSet; } - public RemoteStorePinnedTimestampsBlobStore pinnedTimestampsBlobStore() { - return pinnedTimestampsBlobStore; - } - - public BlobStoreTransferService blobStoreTransferService() { - return blobStoreTransferService; - } - /** * Inner class for asynchronously updating the pinned timestamp set. */ @@ -270,32 +250,22 @@ protected boolean mustReschedule() { @Override protected void runInternal() { long triggerTimestamp = System.currentTimeMillis(); - RemotePinnedTimestamps remotePinnedTimestamps = new RemotePinnedTimestamps( - clusterService.state().metadata().clusterUUID(), - blobStoreRepository.getCompressor() - ); - BlobPath path = pinnedTimestampsBlobStore().getBlobPathForUpload(remotePinnedTimestamps); - blobStoreTransferService().listAllInSortedOrder(path, remotePinnedTimestamps.getType(), 1, new ActionListener<>() { - @Override - public void onResponse(List blobMetadata) { - if (blobMetadata.isEmpty()) { - pinnedTimestampsSet = new Tuple<>(triggerTimestamp, Set.of()); - return; - } - PinnedTimestamps pinnedTimestamps = readExistingPinnedTimestamps(blobMetadata.get(0).name(), remotePinnedTimestamps); - logger.debug( - "Fetched pinned timestamps from remote store: {} - {}", - triggerTimestamp, - pinnedTimestamps.getPinnedTimestampPinningEntityMap().keySet() - ); - pinnedTimestampsSet = new Tuple<>(triggerTimestamp, pinnedTimestamps.getPinnedTimestampPinningEntityMap().keySet()); - } - - @Override - public void onFailure(Exception e) { - logger.error("Exception while listing pinned timestamp files", e); + try { + Map pinnedTimestampList = blobContainer.listBlobs(); + if (pinnedTimestampList.isEmpty()) { + pinnedTimestampsSet = new Tuple<>(triggerTimestamp, Set.of()); + return; } - }); + Set pinnedTimestamps = pinnedTimestampList.keySet() + .stream() + .map(RemoteStorePinnedTimestampService.this::getTimestampFromBlobName) + .filter(timestamp -> timestamp != -1) + .collect(Collectors.toSet()); + logger.debug("Fetched pinned timestamps from remote store: {} - {}", triggerTimestamp, pinnedTimestamps); + pinnedTimestampsSet = new Tuple<>(triggerTimestamp, pinnedTimestamps); + } catch (Throwable t) { + logger.error("Exception while fetching pinned timestamp details", t); + } } } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePinnedTimestampsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePinnedTimestampsTests.java deleted file mode 100644 index 309263a634265..0000000000000 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePinnedTimestampsTests.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.common.compress.DeflateCompressor; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.io.stream.BytesStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.compress.Compressor; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class RemotePinnedTimestampsTests extends OpenSearchTestCase { - - private RemotePinnedTimestamps remotePinnedTimestamps; - - @Before - public void setup() { - Compressor compressor = new DeflateCompressor(); - remotePinnedTimestamps = new RemotePinnedTimestamps("testClusterUUID", compressor); - } - - public void testGenerateBlobFileName() { - String fileName = remotePinnedTimestamps.generateBlobFileName(); - assertTrue(fileName.startsWith(RemotePinnedTimestamps.PINNED_TIMESTAMPS)); - assertEquals(fileName, remotePinnedTimestamps.getBlobFileName()); - } - - public void testSerializeAndDeserialize() throws IOException { - RemotePinnedTimestamps.PinnedTimestamps pinnedTimestamps = new RemotePinnedTimestamps.PinnedTimestamps(new HashMap<>()); - pinnedTimestamps.pin(1000L, "entity1"); - pinnedTimestamps.pin(2000L, "entity2"); - remotePinnedTimestamps.setPinnedTimestamps(pinnedTimestamps); - - InputStream serialized = remotePinnedTimestamps.serialize(); - RemotePinnedTimestamps.PinnedTimestamps deserialized = remotePinnedTimestamps.deserialize(serialized); - - assertEquals(pinnedTimestamps.getPinnedTimestampPinningEntityMap(), deserialized.getPinnedTimestampPinningEntityMap()); - } - - public void testSetAndGetPinnedTimestamps() { - RemotePinnedTimestamps.PinnedTimestamps pinnedTimestamps = new RemotePinnedTimestamps.PinnedTimestamps(new HashMap<>()); - remotePinnedTimestamps.setPinnedTimestamps(pinnedTimestamps); - assertEquals(pinnedTimestamps, remotePinnedTimestamps.getPinnedTimestamps()); - } - - public void testPinnedTimestampsPin() { - RemotePinnedTimestamps.PinnedTimestamps pinnedTimestamps = new RemotePinnedTimestamps.PinnedTimestamps(new HashMap<>()); - pinnedTimestamps.pin(1000L, "entity1"); - pinnedTimestamps.pin(1000L, "entity2"); - pinnedTimestamps.pin(2000L, "entity3"); - - Map> expected = new HashMap<>(); - expected.put(1000L, Arrays.asList("entity1", "entity2")); - expected.put(2000L, List.of("entity3")); - - assertEquals(expected, pinnedTimestamps.getPinnedTimestampPinningEntityMap()); - } - - public void testPinnedTimestampsUnpin() { - RemotePinnedTimestamps.PinnedTimestamps pinnedTimestamps = new RemotePinnedTimestamps.PinnedTimestamps(new HashMap<>()); - pinnedTimestamps.pin(1000L, "entity1"); - pinnedTimestamps.pin(1000L, "entity2"); - pinnedTimestamps.pin(2000L, "entity3"); - - pinnedTimestamps.unpin(1000L, "entity1"); - pinnedTimestamps.unpin(2000L, "entity3"); - - Map> expected = new HashMap<>(); - expected.put(1000L, List.of("entity2")); - - assertEquals(expected, pinnedTimestamps.getPinnedTimestampPinningEntityMap()); - } - - public void testPinnedTimestampsReadFromAndWriteTo() throws IOException { - RemotePinnedTimestamps.PinnedTimestamps original = new RemotePinnedTimestamps.PinnedTimestamps(new HashMap<>()); - original.pin(1000L, "entity1"); - original.pin(2000L, "entity2"); - - BytesStreamOutput out = new BytesStreamOutput(); - original.writeTo(out); - - StreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - RemotePinnedTimestamps.PinnedTimestamps deserialized = RemotePinnedTimestamps.PinnedTimestamps.readFrom(in); - - assertEquals(original.getPinnedTimestampPinningEntityMap(), deserialized.getPinnedTimestampPinningEntityMap()); - } -} diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java index 107d59aa97549..e71023125d4cd 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java @@ -8,15 +8,12 @@ package org.opensearch.index.store; -import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.model.RemotePinnedTimestamps; -import org.opensearch.gateway.remote.model.RemoteStorePinnedTimestampsBlobStore; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -26,7 +23,7 @@ import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -38,7 +35,6 @@ import static org.hamcrest.CoreMatchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.anyInt; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -48,8 +44,7 @@ public class RemoteSegmentStoreDirectoryWithPinnedTimestampTests extends RemoteSegmentStoreDirectoryTests { Runnable updatePinnedTimstampTask; - BlobStoreTransferService blobStoreTransferService; - RemoteStorePinnedTimestampsBlobStore remoteStorePinnedTimestampsBlobStore; + BlobContainer blobContainer; RemoteStorePinnedTimestampService remoteStorePinnedTimestampServiceSpy; @Before @@ -82,16 +77,13 @@ public void setupPinnedTimestamp() throws IOException { ); remoteStorePinnedTimestampServiceSpy = Mockito.spy(remoteStorePinnedTimestampService); - remoteStorePinnedTimestampsBlobStore = mock(RemoteStorePinnedTimestampsBlobStore.class); - blobStoreTransferService = mock(BlobStoreTransferService.class); - when(remoteStorePinnedTimestampServiceSpy.pinnedTimestampsBlobStore()).thenReturn(remoteStorePinnedTimestampsBlobStore); - when(remoteStorePinnedTimestampServiceSpy.blobStoreTransferService()).thenReturn(blobStoreTransferService); + BlobStore blobStore = mock(BlobStore.class); + when(blobStoreRepository.blobStore()).thenReturn(blobStore); + when(blobStoreRepository.basePath()).thenReturn(new BlobPath()); + blobContainer = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).thenReturn(blobContainer); - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(new ArrayList<>()); - return null; - }).when(blobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + when(blobContainer.listBlobs()).thenReturn(new HashMap<>()); remoteStorePinnedTimestampServiceSpy.start(); @@ -195,15 +187,9 @@ public void testDeleteStaleCommitsPinnedTimestampMdFile() throws Exception { ) ).thenReturn(List.of(metadataFilename, metadataFilename2, metadataFilename3)); - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); - return null; - }).when(blobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); - long pinnedTimestampMatchingMetadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getTimestamp(metadataFilename2) + 10; - when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn(new RemotePinnedTimestamps.PinnedTimestamps(Map.of(pinnedTimestampMatchingMetadataFilename2, List.of("xyz")))); - when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + String blobName = "snapshot1__" + pinnedTimestampMatchingMetadataFilename2; + when(blobContainer.listBlobs()).thenReturn(Map.of(blobName, new PlainBlobMetadata(blobName, 100))); final Map> metadataFilenameContentMapping = populateMetadata(); final List filesToBeDeleted = metadataFilenameContentMapping.get(metadataFilename3) diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java index e35ab6bcd14c3..386dde4dffc48 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java @@ -12,8 +12,10 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; @@ -22,12 +24,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.model.RemotePinnedTimestamps; -import org.opensearch.gateway.remote.model.RemoteStorePinnedTimestampsBlobStore; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.index.translog.transfer.TranslogTransferManager; import org.opensearch.index.translog.transfer.TranslogTransferMetadata; import org.opensearch.indices.DefaultRemoteStoreSettings; @@ -63,8 +62,6 @@ import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -73,8 +70,7 @@ public class RemoteFsTranslogWithPinnedTimestampTests extends RemoteFsTranslogTests { Runnable updatePinnedTimstampTask; - BlobStoreTransferService pinnedTimestampBlobStoreTransferService; - RemoteStorePinnedTimestampsBlobStore remoteStorePinnedTimestampsBlobStore; + BlobContainer blobContainer; RemoteStorePinnedTimestampService remoteStorePinnedTimestampServiceSpy; @Before @@ -118,16 +114,13 @@ public void setUp() throws Exception { ); remoteStorePinnedTimestampServiceSpy = Mockito.spy(remoteStorePinnedTimestampService); - remoteStorePinnedTimestampsBlobStore = mock(RemoteStorePinnedTimestampsBlobStore.class); - pinnedTimestampBlobStoreTransferService = mock(BlobStoreTransferService.class); - when(remoteStorePinnedTimestampServiceSpy.pinnedTimestampsBlobStore()).thenReturn(remoteStorePinnedTimestampsBlobStore); - when(remoteStorePinnedTimestampServiceSpy.blobStoreTransferService()).thenReturn(pinnedTimestampBlobStoreTransferService); + BlobStore blobStore = mock(BlobStore.class); + when(blobStoreRepository.blobStore()).thenReturn(blobStore); + when(blobStoreRepository.basePath()).thenReturn(new BlobPath()); + blobContainer = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).thenReturn(blobContainer); - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(new ArrayList<>()); - return null; - }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); + when(blobContainer.listBlobs()).thenReturn(new HashMap<>()); remoteStorePinnedTimestampServiceSpy.start(); } @@ -362,20 +355,12 @@ public void onResponse(List blobMetadataList) { } } - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); - return null; - }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); - - Map> pinnedTimestampsMap = new HashMap<>(); - pinnedTimestamps.forEach(ts -> pinnedTimestampsMap.put(ts, new ArrayList<>())); + Map pinnedTimestampsMap = new HashMap<>(); + pinnedTimestamps.forEach(ts -> pinnedTimestampsMap.put(randomInt(1000) + "__" + ts, new PlainBlobMetadata("x", 100))); try { - when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( - new RemotePinnedTimestamps.PinnedTimestamps(pinnedTimestampsMap) - ); - when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + + when(blobContainer.listBlobs()).thenReturn(pinnedTimestampsMap); Set dataFilesBeforeTrim = blobStoreTransferService.listAll( getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get())) @@ -642,7 +627,7 @@ public void testGetMetadataFilesToBeDeletedNoExclusion() { public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { updatePinnedTimstampTask.run(); long currentTimeInMillis = System.currentTimeMillis(); - String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 100000); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000); String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000); @@ -659,21 +644,12 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws IOException { long currentTimeInMillis = System.currentTimeMillis(); - String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 100000); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); - return null; - }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); - long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000; - when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( - new RemotePinnedTimestamps.PinnedTimestamps(Map.of(pinnedTimestamp, List.of("xyz"))) - ); - when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + when(blobContainer.listBlobs()).thenReturn(Map.of(randomInt(100) + "__" + pinnedTimestamp, new PlainBlobMetadata("xyz", 100))); updatePinnedTimstampTask.run(); @@ -695,17 +671,8 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throw String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); - doAnswer(invocationOnMock -> { - ActionListener> actionListener = invocationOnMock.getArgument(3); - actionListener.onResponse(List.of(new PlainBlobMetadata("pinned_timestamp_123", 1000))); - return null; - }).when(pinnedTimestampBlobStoreTransferService).listAllInSortedOrder(any(), any(), eq(1), any()); - long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000; - when(remoteStorePinnedTimestampsBlobStore.read(any())).thenReturn( - new RemotePinnedTimestamps.PinnedTimestamps(Map.of(pinnedTimestamp, List.of("xyz"))) - ); - when(remoteStorePinnedTimestampsBlobStore.getBlobPathForUpload(any())).thenReturn(new BlobPath()); + when(blobContainer.listBlobs()).thenReturn(Map.of(randomInt(100) + "__" + pinnedTimestamp, new PlainBlobMetadata("xyz", 100))); updatePinnedTimstampTask.run(); From 41ba00a0300e2e9c62ebf2d88502a97c7189738c Mon Sep 17 00:00:00 2001 From: anand kumar rai Date: Tue, 3 Sep 2024 09:53:01 +0530 Subject: [PATCH 011/357] Add limit on number of processors in Ingest pipelines (#15465) * Add limit on number of processors in Ingest pipelines Signed-off-by: Rai --- CHANGELOG.md | 1 + .../ingest/SimulateExecutionService.java | 8 +- .../SimulatePipelineTransportAction.java | 2 +- .../common/settings/ClusterSettings.java | 2 + .../org/opensearch/ingest/IngestService.java | 37 +++++ .../ingest/SimulateExecutionServiceTests.java | 23 ++- .../opensearch/ingest/IngestServiceTests.java | 155 +++++++++++++++++- 7 files changed, 221 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 442677287daf3..f17741c232b1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) - Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) - [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) +- Add limit on number of processors for Ingest pipeline([#15460](https://github.com/opensearch-project/OpenSearch/pull/15465)). - Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630)) - Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454)) - [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/opensearch/action/ingest/SimulateExecutionService.java index c7c0f21eb0876..459466f8c8ab6 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulateExecutionService.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulateExecutionService.java @@ -36,6 +36,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.ingest.CompoundProcessor; import org.opensearch.ingest.IngestDocument; +import org.opensearch.ingest.IngestService; import org.opensearch.ingest.Pipeline; import org.opensearch.threadpool.ThreadPool; @@ -56,9 +57,11 @@ class SimulateExecutionService { private static final String THREAD_POOL_NAME = ThreadPool.Names.MANAGEMENT; private final ThreadPool threadPool; + private final IngestService ingestService; - SimulateExecutionService(ThreadPool threadPool) { + SimulateExecutionService(ThreadPool threadPool, IngestService ingestService) { this.threadPool = threadPool; + this.ingestService = ingestService; } void executeDocument( @@ -91,6 +94,9 @@ void executeDocument( } public void execute(SimulatePipelineRequest.Parsed request, ActionListener listener) { + + ingestService.validateProcessorCountForIngestPipeline(request.getPipeline()); + threadPool.executor(THREAD_POOL_NAME).execute(ActionRunnable.wrap(listener, l -> { final AtomicInteger counter = new AtomicInteger(); final List responses = new CopyOnWriteArrayList<>( diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java index 4753679d370af..5eeb09c4d50c0 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineTransportAction.java @@ -69,7 +69,7 @@ public SimulatePipelineTransportAction( (Writeable.Reader) SimulatePipelineRequest::new ); this.ingestService = ingestService; - this.executionService = new SimulateExecutionService(threadPool); + this.executionService = new SimulateExecutionService(threadPool, ingestService); } @Override diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 9a6b3f1118709..31e47dc14db23 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -132,6 +132,7 @@ import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.store.IndicesStore; +import org.opensearch.ingest.IngestService; import org.opensearch.monitor.fs.FsHealthService; import org.opensearch.monitor.fs.FsService; import org.opensearch.monitor.jvm.JvmGcMonitorService; @@ -406,6 +407,7 @@ public void apply(Settings value, Settings current, Settings previous) { ClusterService.USER_DEFINED_METADATA, ClusterManagerService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING, // deprecated ClusterManagerService.CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING, + IngestService.MAX_NUMBER_OF_INGEST_PROCESSORS, SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING, SearchService.DEFAULT_ALLOW_PARTIAL_SEARCH_RESULTS, TransportSearchAction.SHARD_COUNT_LIMIT_SETTING, diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java index 938ca7493926e..0315a960dae92 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestService.java +++ b/server/src/main/java/org/opensearch/ingest/IngestService.java @@ -62,6 +62,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.metrics.OperationMetrics; import org.opensearch.common.regex.Regex; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; @@ -107,6 +108,18 @@ public class IngestService implements ClusterStateApplier, ReportingService MAX_NUMBER_OF_INGEST_PROCESSORS = Setting.intSetting( + "cluster.ingest.max_number_processors", + Integer.MAX_VALUE, + 1, + Integer.MAX_VALUE, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final Logger logger = LogManager.getLogger(IngestService.class); private final ClusterService clusterService; @@ -123,6 +136,7 @@ public class IngestService implements ClusterStateApplier, ReportingService processorFactories(List ingestPlugins, Processor.Parameters parameters) { @@ -494,6 +514,9 @@ void validatePipeline(Map ingestInfos, PutPipelineReq Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2(); Pipeline pipeline = Pipeline.create(request.getId(), pipelineConfig, processorFactories, scriptService); + + validateProcessorCountForIngestPipeline(pipeline); + List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { for (Map.Entry entry : ingestInfos.entrySet()) { @@ -507,6 +530,20 @@ void validatePipeline(Map ingestInfos, PutPipelineReq ExceptionsHelper.rethrowAndSuppress(exceptions); } + public void validateProcessorCountForIngestPipeline(Pipeline pipeline) { + List processors = pipeline.flattenAllProcessors(); + + if (processors.size() > maxIngestProcessorCount) { + throw new IllegalStateException( + "Cannot use more than the maximum processors allowed. Number of processors being configured is [" + + processors.size() + + "] which exceeds the maximum allowed configuration of [" + + maxIngestProcessorCount + + "] processors." + ); + } + } + public void executeBulkRequest( int numberOfActionRequests, Iterable> actionRequests, diff --git a/server/src/test/java/org/opensearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/opensearch/action/ingest/SimulateExecutionServiceTests.java index a5a082286f123..a26afeee0f912 100644 --- a/server/src/test/java/org/opensearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/SimulateExecutionServiceTests.java @@ -39,6 +39,7 @@ import org.opensearch.ingest.DropProcessor; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.IngestProcessorException; +import org.opensearch.ingest.IngestService; import org.opensearch.ingest.Pipeline; import org.opensearch.ingest.Processor; import org.opensearch.ingest.RandomDocumentPicks; @@ -67,6 +68,8 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; public class SimulateExecutionServiceTests extends OpenSearchTestCase { @@ -75,11 +78,13 @@ public class SimulateExecutionServiceTests extends OpenSearchTestCase { private TestThreadPool threadPool; private SimulateExecutionService executionService; private IngestDocument ingestDocument; + private IngestService ingestService; @Before public void setup() { + ingestService = mock(IngestService.class); threadPool = new TestThreadPool(SimulateExecutionServiceTests.class.getSimpleName()); - executionService = new SimulateExecutionService(threadPool); + executionService = new SimulateExecutionService(threadPool, ingestService); ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); } @@ -400,6 +405,22 @@ public String getType() { } } + public void testValidateProcessorCountForIngestPipelineThrowsException() { + + int numDocs = randomIntBetween(1, 64); + List documents = new ArrayList<>(numDocs); + for (int id = 0; id < numDocs; id++) { + documents.add(new IngestDocument("_index", Integer.toString(id), null, 0L, VersionType.INTERNAL, new HashMap<>())); + } + + Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor()); + SimulatePipelineRequest.Parsed request = new SimulatePipelineRequest.Parsed(pipeline, documents, false); + + doThrow(new IllegalStateException()).when(ingestService).validateProcessorCountForIngestPipeline(pipeline); + + expectThrows(IllegalStateException.class, () -> executionService.execute(request, ActionListener.wrap(response -> {}, e -> {}))); + } + private static void assertVerboseResult( SimulateProcessorResult result, String expectedPipelineId, diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java index 1f4b1d635d438..e1e1ea561284b 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java @@ -58,6 +58,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.metrics.OperationStats; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.xcontent.XContentType; @@ -151,8 +152,12 @@ public void setup() { public void testIngestPlugin() { Client client = mock(Client.class); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); IngestService ingestService = new IngestService( - mock(ClusterService.class), + clusterService, threadPool, null, null, @@ -186,8 +191,12 @@ public void testIngestPluginDuplicate() { public void testExecuteIndexPipelineDoesNotExist() { Client client = mock(Client.class); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); IngestService ingestService = new IngestService( - mock(ClusterService.class), + clusterService, threadPool, null, null, @@ -720,6 +729,124 @@ public void testValidate() throws Exception { ingestService.validatePipeline(ingestInfos, putRequest); } + public void testValidateProcessorCountForIngestPipelineThrowsException() { + IngestService ingestService = createWithProcessors(); + PutPipelineRequest putRequest = new PutPipelineRequest( + "_id", + new BytesArray( + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\", \"tag\": \"tag1\"}}," + + "{\"remove\" : {\"field\": \"_field\", \"tag\": \"tag2\"}}]}" + ), + MediaTypeRegistry.JSON + ); + + DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node2 = new DiscoveryNode("_node_id2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + Map ingestInfos = new HashMap<>(); + ingestInfos.put(node1, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove")))); + ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set")))); + + Settings newSettings = Settings.builder().put("cluster.ingest.max_number_processors", 1).build(); + ingestService.getClusterService().getClusterSettings().applySettings(newSettings); + + expectThrows(IllegalStateException.class, () -> ingestService.validatePipeline(ingestInfos, putRequest)); + } + + public void testValidateProcessorCountForWithNestedOnFailureProcessorThrowsException() { + IngestService ingestService = createWithProcessors(); + PutPipelineRequest putRequest = new PutPipelineRequest( + "_id", + new BytesArray( + "{\n" + + " \"processors\": [\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"timestamp_field_1\",\n" + + " \"value\": \"value\",\n" + + " \"on_failure\": [\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error1\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\",\n" + + " \"on_failure\": [\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error1\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\",\n" + + " \"on_failure\": [\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error1\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\",\n" + + " \"on_failure\": [\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error1\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error2\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error2\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error2\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"set\": {\n" + + " \"field\": \"ingest_error2\",\n" + + " \"value\": \"failed\",\n" + + " \"tag\": \"tagggg\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}" + ), + MediaTypeRegistry.JSON + ); + + DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + DiscoveryNode node2 = new DiscoveryNode("_node_id2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + Map ingestInfos = new HashMap<>(); + ingestInfos.put(node1, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove")))); + ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set")))); + + Settings newSettings = Settings.builder().put("cluster.ingest.max_number_processors", 7).build(); + ingestService.getClusterService().getClusterSettings().applySettings(newSettings); + + expectThrows(IllegalStateException.class, () -> ingestService.validatePipeline(ingestInfos, putRequest)); + } + public void testExecuteIndexPipelineExistsButFailedParsing() { IngestService ingestService = createWithProcessors( Collections.singletonMap("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") { @@ -1506,8 +1633,12 @@ public Map getProcessors(Processor.Parameters paramet // Create ingest service: Client client = mock(Client.class); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); IngestService ingestService = new IngestService( - mock(ClusterService.class), + clusterService, threadPool, null, null, @@ -2058,6 +2189,18 @@ public void testPrepareBatches_different_index_pipeline() { assertEquals(4, batches.size()); } + public void testUpdateMaxIngestProcessorCountSetting() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + // verify defaults + assertEquals(Integer.MAX_VALUE, clusterSettings.get(IngestService.MAX_NUMBER_OF_INGEST_PROCESSORS).intValue()); + + // verify update max processor + Settings newSettings = Settings.builder().put("cluster.ingest.max_number_processors", 3).build(); + clusterSettings.applySettings(newSettings); + assertEquals(3, clusterSettings.get(IngestService.MAX_NUMBER_OF_INGEST_PROCESSORS).intValue()); + } + private IngestService.IndexRequestWrapper createIndexRequestWrapper(String index, List pipelines) { IndexRequest indexRequest = new IndexRequest(index); return new IngestService.IndexRequestWrapper(0, indexRequest, pipelines, true); @@ -2093,7 +2236,11 @@ private static IngestService createWithProcessors(Map ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService(); when(threadPool.generic()).thenReturn(executorService); when(threadPool.executor(anyString())).thenReturn(executorService); - return new IngestService(mock(ClusterService.class), threadPool, null, null, null, Collections.singletonList(new IngestPlugin() { + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn( + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + return new IngestService(clusterService, threadPool, null, null, null, Collections.singletonList(new IngestPlugin() { @Override public Map getProcessors(final Processor.Parameters parameters) { return processors; From 925f41b5635aae02b283866832da3aff6f4049b2 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:47:53 +0530 Subject: [PATCH 012/357] [SnapshotV2] Snapshot Status API changes (#15409) --------- Signed-off-by: Lakshya Taragi --- CHANGELOG.md | 1 + .../SnapshotRequestConvertersTests.java | 3 +- .../MultiVersionRepositoryAccessIT.java | 18 +- .../rest-api-spec/api/snapshot.status.json | 22 +- .../RemoteIndexSnapshotStatusApiIT.java | 112 ++++++++++ .../snapshots/SnapshotStatusApisIT.java | 196 ++++++++++++++++++ .../opensearch/OpenSearchServerException.java | 8 + .../status/SnapshotsStatusRequest.java | 43 +++- .../status/SnapshotsStatusRequestBuilder.java | 28 ++- .../TransportSnapshotsStatusAction.java | 189 ++++++++++++++--- .../common/settings/ClusterSettings.java | 1 + .../cluster/RestSnapshotsStatusAction.java | 4 +- .../snapshots/SnapshotsService.java | 12 ++ ...oManyShardsInSnapshotsStatusException.java | 69 ++++++ .../ExceptionSerializationTests.java | 2 + 15 files changed, 657 insertions(+), 51 deletions(-) create mode 100644 server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f17741c232b1a..ff823efef1271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) +- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java index af178ad2a5d47..e9ca03262bc4f 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java @@ -230,6 +230,7 @@ public void testSnapshotsStatus() { Map expectedParams = new HashMap<>(); String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0]; String[] snapshots = RequestConvertersTests.randomIndicesNames(1, 5); + String[] indices = RequestConvertersTests.randomIndicesNames(1, 5); StringBuilder snapshotNames = new StringBuilder(snapshots[0]); for (int idx = 1; idx < snapshots.length; idx++) { snapshotNames.append(",").append(snapshots[idx]); @@ -237,7 +238,7 @@ public void testSnapshotsStatus() { boolean ignoreUnavailable = randomBoolean(); String endpoint = "/_snapshot/" + repository + "/" + snapshotNames.toString() + "/_status"; - SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots); + SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots, indices); RequestConvertersTests.setRandomClusterManagerTimeout(snapshotsStatusRequest, expectedParams); snapshotsStatusRequest.ignoreUnavailable(ignoreUnavailable); expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable)); diff --git a/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java b/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java index 7c27f2ea71942..5c6dcdd142995 100644 --- a/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java +++ b/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java @@ -32,6 +32,7 @@ package org.opensearch.upgrades; +import com.sun.jna.StringArray; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus; @@ -44,6 +45,7 @@ import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; @@ -141,14 +143,14 @@ public void testCreateAndRestoreSnapshot() throws IOException { case STEP2_NEW_CLUSTER: case STEP4_NEW_CLUSTER: assertSnapshotStatusSuccessful(client, repoName, - snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new)); + snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new), Strings.EMPTY_ARRAY); break; case STEP1_OLD_CLUSTER: - assertSnapshotStatusSuccessful(client, repoName, "snapshot-" + TEST_STEP); + assertSnapshotStatusSuccessful(client, repoName, new String[] {"snapshot-" + TEST_STEP}, Strings.EMPTY_ARRAY); break; case STEP3_OLD_CLUSTER: assertSnapshotStatusSuccessful( - client, repoName, "snapshot-" + TEST_STEP, "snapshot-" + TestStep.STEP3_OLD_CLUSTER); + client, repoName, new String[] {"snapshot-" + TEST_STEP, "snapshot-" + TestStep.STEP3_OLD_CLUSTER}, Strings.EMPTY_ARRAY); break; } if (TEST_STEP == TestStep.STEP3_OLD_CLUSTER) { @@ -186,10 +188,10 @@ public void testReadOnlyRepo() throws IOException { break; } if (TEST_STEP == TestStep.STEP1_OLD_CLUSTER || TEST_STEP == TestStep.STEP3_OLD_CLUSTER) { - assertSnapshotStatusSuccessful(client, repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER); + assertSnapshotStatusSuccessful(client, repoName, new String[] {"snapshot-" + TestStep.STEP1_OLD_CLUSTER}, Strings.EMPTY_ARRAY); } else { assertSnapshotStatusSuccessful(client, repoName, - "snapshot-" + TestStep.STEP1_OLD_CLUSTER, "snapshot-" + TestStep.STEP2_NEW_CLUSTER); + new String[] {"snapshot-" + TestStep.STEP1_OLD_CLUSTER, "snapshot-" + TestStep.STEP2_NEW_CLUSTER}, Strings.EMPTY_ARRAY); } if (TEST_STEP == TestStep.STEP3_OLD_CLUSTER) { ensureSnapshotRestoreWorks(repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER, shards); @@ -214,7 +216,7 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException { // Every step creates one snapshot assertThat(snapshots, hasSize(TEST_STEP.ordinal() + 1)); assertSnapshotStatusSuccessful(client, repoName, - snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new)); + snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new), Strings.EMPTY_ARRAY); if (TEST_STEP == TestStep.STEP1_OLD_CLUSTER) { ensureSnapshotRestoreWorks(repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER, shards); } else { @@ -239,9 +241,9 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException { } private static void assertSnapshotStatusSuccessful(RestHighLevelClient client, String repoName, - String... snapshots) throws IOException { + String[] snapshots, String[] indices) throws IOException { final SnapshotsStatusResponse statusResponse = client.snapshot() - .status(new SnapshotsStatusRequest(repoName, snapshots), RequestOptions.DEFAULT); + .status(new SnapshotsStatusRequest(repoName, snapshots, indices), RequestOptions.DEFAULT); for (SnapshotStatus status : statusResponse.getSnapshots()) { assertThat(status.getShardsStats().getFailedShards(), is(0)); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json index 1ac6042941013..354d3c35d2bda 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json @@ -40,6 +40,26 @@ "description":"A comma-separated list of snapshot names" } } + }, + { + "path":"/_snapshot/{repository}/{snapshot}/{index}/_status", + "methods":[ + "GET" + ], + "parts":{ + "repository":{ + "type":"string", + "description":"A repository name" + }, + "snapshot":{ + "type":"string", + "description":"A snapshot name" + }, + "index":{ + "type": "list", + "description":"A comma-separated list of index names" + } + } } ] }, @@ -58,7 +78,7 @@ }, "ignore_unavailable":{ "type":"boolean", - "description":"Whether to ignore unavailable snapshots, defaults to false which means a SnapshotMissingException is thrown" + "description":"Whether to ignore unavailable snapshots and indices, defaults to false which means a SnapshotMissingException or IndexNotFoundException is thrown" } } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java index 8e2580aba1745..e84de36df2fca 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -32,20 +32,28 @@ package org.opensearch.snapshots; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus; +import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; import org.junit.Before; import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -192,6 +200,110 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception { createSnapshotResponseActionFuture.actionGet(); } + public void testStatusAPICallForShallowV2Snapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used for the test"); + Settings pinnedTimestampSettings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings); + internalCluster().startDataOnlyNodes(2, pinnedTimestampSettings); + + final String index1 = "remote-index-1"; + final String index2 = "remote-index-2"; + final String index3 = "remote-index-3"; + final String snapshotRepoName = "snapshot-repo-name"; + final String snapshot = "snapshot"; + + logger.info("Create repository for shallow V2 snapshots"); + Settings.Builder snapshotV2RepoSettings = snapshotRepoSettingsForShallowCopy().put( + BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), + Boolean.TRUE + ); + createRepository(snapshotRepoName, "fs", snapshotV2RepoSettings); + + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(index1, remoteStoreEnabledIndexSettings); + createIndex(index2, remoteStoreEnabledIndexSettings); + createIndex(index3, remoteStoreEnabledIndexSettings); + ensureGreen(); + + logger.info("Indexing some data"); + for (int i = 0; i < 50; i++) { + index(index1, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index2, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index3, "_doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + + SnapshotInfo snapshotInfo = createFullSnapshot(snapshotRepoName, snapshot); + assertTrue(snapshotInfo.getPinnedTimestamp() > 0); // to assert creation of a shallow v2 snapshot + + logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value"); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + assertBusy(() -> { + // without index filter + // although no. of shards in snapshot (3) is greater than the max value allowed in a status api call, the request does not fail + SnapshotStatus snapshotStatusWithoutIndexFilter = client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots(snapshot) + .execute() + .actionGet() + .getSnapshots() + .get(0); + + assertShallowV2SnapshotStatus(snapshotStatusWithoutIndexFilter, false); + + // with index filter + SnapshotStatus snapshotStatusWithIndexFilter = client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots(snapshot) + .setIndices(index1, index2) + .execute() + .actionGet() + .getSnapshots() + .get(0); + + assertShallowV2SnapshotStatus(snapshotStatusWithIndexFilter, true); + + }, 1, TimeUnit.MINUTES); + + } + + private void assertShallowV2SnapshotStatus(SnapshotStatus snapshotStatus, boolean hasIndexFilter) { + if (hasIndexFilter) { + assertEquals(0, snapshotStatus.getStats().getTotalSize()); + } else { + // TODO: after adding primary store size at the snapshot level, total size here should be > 0 + } + // assert that total and incremental values of file count and size_in_bytes are 0 at index and shard levels + assertEquals(0, snapshotStatus.getStats().getTotalFileCount()); + assertEquals(0, snapshotStatus.getStats().getIncrementalSize()); + assertEquals(0, snapshotStatus.getStats().getIncrementalFileCount()); + + for (Map.Entry entry : snapshotStatus.getIndices().entrySet()) { + // index level + SnapshotIndexStatus snapshotIndexStatus = entry.getValue(); + assertEquals(0, snapshotIndexStatus.getStats().getTotalSize()); + assertEquals(0, snapshotIndexStatus.getStats().getTotalFileCount()); + assertEquals(0, snapshotIndexStatus.getStats().getIncrementalSize()); + assertEquals(0, snapshotIndexStatus.getStats().getIncrementalFileCount()); + + for (SnapshotIndexShardStatus snapshotIndexShardStatus : snapshotStatus.getShards()) { + // shard level + assertEquals(0, snapshotIndexShardStatus.getStats().getTotalSize()); + assertEquals(0, snapshotIndexShardStatus.getStats().getTotalFileCount()); + assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalSize()); + assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalFileCount()); + assertEquals(SnapshotIndexShardStage.DONE, snapshotIndexShardStatus.getStage()); + } + } + } + private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) { return snapshotStatus.getIndices().get(indexName).getShards().get(0); } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index 3c75b30580c30..5a043e69e9735 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -33,11 +33,14 @@ package org.opensearch.snapshots; import org.opensearch.Version; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus; +import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; @@ -49,6 +52,8 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -60,9 +65,12 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API; import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -564,6 +572,194 @@ public void testGetSnapshotsRequest() throws Exception { waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60)); } + public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws Exception { + String repositoryName = "test-repo"; + String index1 = "test-idx-1"; + String index2 = "test-idx-2"; + String index3 = "test-idx-3"; + createRepository(repositoryName, "fs"); + + logger.info("Create indices"); + createIndex(index1, index2, index3); + ensureGreen(); + + logger.info("Indexing some data"); + for (int i = 0; i < 10; i++) { + index(index1, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index2, "_doc", Integer.toString(i), "foo", "baz" + i); + index(index3, "_doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + String snapshot1 = "test-snap-1"; + String snapshot2 = "test-snap-2"; + createSnapshot(repositoryName, snapshot1, List.of(index1, index2, index3)); + createSnapshot(repositoryName, snapshot2, List.of(index1, index2)); + + logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value"); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // across a single snapshot + assertBusy(() -> { + TooManyShardsInSnapshotsStatusException exception = expectThrows( + TooManyShardsInSnapshotsStatusException.class, + () -> client().admin().cluster().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot1).execute().actionGet() + ); + assertEquals(exception.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertTrue( + exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") + ); + }, 1, TimeUnit.MINUTES); + + // across multiple snapshots + assertBusy(() -> { + TooManyShardsInSnapshotsStatusException exception = expectThrows( + TooManyShardsInSnapshotsStatusException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot1, snapshot2) + .execute() + .actionGet() + ); + assertEquals(exception.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertTrue( + exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") + ); + }, 1, TimeUnit.MINUTES); + + logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); + updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey())); + assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + } + + public void testSnapshotStatusForIndexFilter() throws Exception { + String repositoryName = "test-repo"; + String index1 = "test-idx-1"; + String index2 = "test-idx-2"; + String index3 = "test-idx-3"; + createRepository(repositoryName, "fs"); + + logger.info("Create indices"); + createIndex(index1, index2, index3); + ensureGreen(); + + logger.info("Indexing some data"); + for (int i = 0; i < 10; i++) { + index(index1, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index2, "_doc", Integer.toString(i), "foo", "baz" + i); + index(index3, "_doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + String snapshot = "test-snap-1"; + createSnapshot(repositoryName, snapshot, List.of(index1, index2, index3)); + + assertBusy(() -> { + SnapshotStatus snapshotsStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot) + .setIndices(index1, index2) + .get() + .getSnapshots() + .get(0); + Map snapshotIndexStatusMap = snapshotsStatus.getIndices(); + // Although the snapshot contains 3 indices, the response of status api call only contains results for 2 + assertEquals(snapshotIndexStatusMap.size(), 2); + assertEquals(snapshotIndexStatusMap.keySet(), Set.of(index1, index2)); + }, 1, TimeUnit.MINUTES); + } + + public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { + String repositoryName = "test-repo"; + String index1 = "test-idx-1"; + String index2 = "test-idx-2"; + String index3 = "test-idx-3"; + createRepository(repositoryName, "fs"); + + logger.info("Create indices"); + createIndex(index1, index2, index3); + ensureGreen(); + + logger.info("Indexing some data"); + for (int i = 0; i < 10; i++) { + index(index1, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index2, "_doc", Integer.toString(i), "foo", "baz" + i); + index(index3, "_doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + String snapshot1 = "test-snap-1"; + String snapshot2 = "test-snap-2"; + createSnapshot(repositoryName, snapshot1, List.of(index1, index2, index3)); + createSnapshot(repositoryName, snapshot2, List.of(index1)); + + assertBusy(() -> { + // failure due to passing index filter for multiple snapshots + ActionRequestValidationException ex = expectThrows( + ActionRequestValidationException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot1, snapshot2) + .setIndices(index1, index2, index3) + .execute() + .actionGet() + ); + String cause = "index list filter is supported only for a single snapshot"; + assertTrue(ex.getMessage().contains(cause)); + }, 1, TimeUnit.MINUTES); + + assertBusy(() -> { + // failure due to index not found in snapshot + IndexNotFoundException ex = expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot2) + .setIndices(index1, index2, index3) + .execute() + .actionGet() + ); + assertEquals(ex.status(), RestStatus.NOT_FOUND); + String cause = String.format( + Locale.ROOT, + "indices [%s] missing in snapshot [%s] of repository [%s]", + String.join(", ", List.of(index2, index3)), + snapshot2, + repositoryName + ); + assertEquals(cause, ex.getCause().getMessage()); + + }, 1, TimeUnit.MINUTES); + + assertBusy(() -> { + // failure due to too many shards requested + logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value"); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + TooManyShardsInSnapshotsStatusException ex = expectThrows( + TooManyShardsInSnapshotsStatusException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot1) + .setIndices(index1, index2, index3) + .execute() + .actionGet() + ); + assertEquals(ex.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertTrue(ex.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request")); + + logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); + updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey())); + assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + }, 2, TimeUnit.MINUTES); + } + private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) { return snapshotStatus.getIndices().get(indexName).getShards().get(0); } diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index b0ab62259d5c0..cc3ceeb7942d3 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -1210,6 +1210,14 @@ public static void registerExceptions() { V_2_17_0 ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class, + org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new, + 175, + V_3_0_0 + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index 061e73f1094b5..87e863f8dd286 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.snapshots.status; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.common.annotation.PublicApi; @@ -54,6 +55,7 @@ public class SnapshotsStatusRequest extends ClusterManagerNodeRequesttrue to ignore unavailable snapshots, instead of throwing an exception. - * Defaults to false, which means unavailable snapshots cause an exception to be thrown. + * Returns the names of the indices. + * + * @return the names of indices + */ + public String[] indices() { + return this.indices; + } + + /** + * Sets the list of indices to be returned + * + * @return this request + */ + public SnapshotsStatusRequest indices(String[] indices) { + this.indices = indices; + return this; + } + + /** + * Set to true to ignore unavailable snapshots and indices, instead of throwing an exception. + * Defaults to false, which means unavailable snapshots and indices cause an exception to be thrown. * - * @param ignoreUnavailable whether to ignore unavailable snapshots + * @param ignoreUnavailable whether to ignore unavailable snapshots and indices * @return this request */ public SnapshotsStatusRequest ignoreUnavailable(boolean ignoreUnavailable) { @@ -158,9 +189,9 @@ public SnapshotsStatusRequest ignoreUnavailable(boolean ignoreUnavailable) { } /** - * Returns whether the request permits unavailable snapshots to be ignored. + * Returns whether the request permits unavailable snapshots and indices to be ignored. * - * @return true if the request will ignore unavailable snapshots, false if it will throw an exception on unavailable snapshots + * @return true if the request will ignore unavailable snapshots and indices, false if it will throw an exception on unavailable snapshots and indices */ public boolean ignoreUnavailable() { return ignoreUnavailable; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java index 9377eca60e353..6f0ac278d01c4 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequestBuilder.java @@ -96,10 +96,32 @@ public SnapshotsStatusRequestBuilder addSnapshots(String... snapshots) { } /** - * Set to true to ignore unavailable snapshots, instead of throwing an exception. - * Defaults to false, which means unavailable snapshots cause an exception to be thrown. + * Sets list of indices to return * - * @param ignoreUnavailable whether to ignore unavailable snapshots. + * @param indices list of indices + * @return this builder + */ + public SnapshotsStatusRequestBuilder setIndices(String... indices) { + request.indices(indices); + return this; + } + + /** + * Adds additional indices to the list of indices to return + * + * @param indices additional indices + * @return this builder + */ + public SnapshotsStatusRequestBuilder addIndices(String... indices) { + request.indices(ArrayUtils.concat(request.indices(), indices)); + return this; + } + + /** + * Set to true to ignore unavailable snapshots and indices, instead of throwing an exception. + * Defaults to false, which means unavailable snapshots and indices cause an exception to be thrown. + * + * @param ignoreUnavailable whether to ignore unavailable snapshots and indices. * @return this builder */ public SnapshotsStatusRequestBuilder setIgnoreUnavailable(boolean ignoreUnavailable) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 4fc2acb2caa51..f2a9b88f790c9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -52,6 +52,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; @@ -65,6 +66,7 @@ import org.opensearch.snapshots.SnapshotShardsService; import org.opensearch.snapshots.SnapshotState; import org.opensearch.snapshots.SnapshotsService; +import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -81,6 +83,7 @@ import java.util.stream.Collectors; import static java.util.Collections.unmodifiableMap; +import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API; /** * Transport action for accessing snapshot status @@ -95,6 +98,8 @@ public class TransportSnapshotsStatusAction extends TransportClusterManagerNodeA private final TransportNodesSnapshotsStatus transportNodesSnapshotsStatus; + private long maximumAllowedShardCount; + @Inject public TransportSnapshotsStatusAction( TransportService transportService, @@ -314,38 +319,34 @@ private void loadRepositoryData( String repositoryName, ActionListener listener ) { - final Set requestedSnapshotNames = Sets.newHashSet(request.snapshots()); + maximumAllowedShardCount = clusterService.getClusterSettings().get(MAX_SHARDS_ALLOWED_IN_STATUS_API); final StepListener repositoryDataListener = new StepListener<>(); repositoriesService.getRepositoryData(repositoryName, repositoryDataListener); repositoryDataListener.whenComplete(repositoryData -> { - final Map matchedSnapshotIds = repositoryData.getSnapshotIds() - .stream() - .filter(s -> requestedSnapshotNames.contains(s.getName())) - .collect(Collectors.toMap(SnapshotId::getName, Function.identity())); - for (final String snapshotName : request.snapshots()) { - if (currentSnapshotNames.contains(snapshotName)) { - // we've already found this snapshot in the current snapshot entries, so skip over - continue; - } - SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName); - if (snapshotId == null) { - // neither in the current snapshot entries nor found in the repository - if (request.ignoreUnavailable()) { - // ignoring unavailable snapshots, so skip over - logger.debug( - "snapshot status request ignoring snapshot [{}], not found in repository [{}]", - snapshotName, - repositoryName - ); - continue; - } else { - throw new SnapshotMissingException(repositoryName, snapshotName); - } - } - SnapshotInfo snapshotInfo = snapshot(snapshotsInProgress, repositoryName, snapshotId); + Map snapshotsInfoMap = snapshotsInfo( + request, + repositoryName, + repositoryData, + snapshotsInProgress, + currentSnapshotNames + ); + for (Map.Entry entry : snapshotsInfoMap.entrySet()) { + SnapshotId snapshotId = entry.getKey(); + SnapshotInfo snapshotInfo = entry.getValue(); List shardStatusBuilder = new ArrayList<>(); if (snapshotInfo.state().completed()) { - Map shardStatuses = snapshotShards(repositoryName, repositoryData, snapshotInfo); + Map shardStatuses = snapshotShards( + request, + repositoryName, + repositoryData, + snapshotInfo + ); + boolean isShallowV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; + long initialSnapshotTotalSize = 0; + if (isShallowV2Snapshot && request.indices().length == 0) { + // TODO: add primary store size in bytes at the snapshot level + } + for (Map.Entry shardStatus : shardStatuses.entrySet()) { IndexShardSnapshotStatus.Copy lastSnapshotStatus = shardStatus.getValue().asCopy(); shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), lastSnapshotStatus)); @@ -406,6 +407,68 @@ private SnapshotInfo snapshot(SnapshotsInProgress snapshotsInProgress, String re return repositoriesService.repository(repositoryName).getSnapshotInfo(snapshotId); } + /** + * Returns snapshot info for finished snapshots + * @param request snapshot status request + * @param repositoryName repository name + * @param repositoryData repository data + * @param snapshotsInProgress currently running snapshots + * @param currentSnapshotNames list of names of currently running snapshots + * @return map of snapshot id to snapshot info + */ + private Map snapshotsInfo( + SnapshotsStatusRequest request, + String repositoryName, + RepositoryData repositoryData, + SnapshotsInProgress snapshotsInProgress, + Set currentSnapshotNames + ) { + final Set requestedSnapshotNames = Sets.newHashSet(request.snapshots()); + final Map snapshotsInfoMap = new HashMap<>(); + final Map matchedSnapshotIds = repositoryData.getSnapshotIds() + .stream() + .filter(s -> requestedSnapshotNames.contains(s.getName())) + .collect(Collectors.toMap(SnapshotId::getName, Function.identity())); + int totalShardsAcrossSnapshots = 0; + for (final String snapshotName : request.snapshots()) { + if (currentSnapshotNames.contains(snapshotName)) { + // we've already found this snapshot in the current snapshot entries, so skip over + continue; + } + SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName); + if (snapshotId == null) { + // neither in the current snapshot entries nor found in the repository + if (request.ignoreUnavailable()) { + // ignoring unavailable snapshots, so skip over + logger.debug( + "snapshot status request ignoring snapshot [{}], not found in repository [{}]", + snapshotName, + repositoryName + ); + continue; + } else { + throw new SnapshotMissingException(repositoryName, snapshotName); + } + } + SnapshotInfo snapshotInfo = snapshot(snapshotsInProgress, repositoryName, snapshotId); + boolean isV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; + if (isV2Snapshot == false && request.indices().length == 0) { + totalShardsAcrossSnapshots += snapshotInfo.totalShards(); + } + snapshotsInfoMap.put(snapshotId, snapshotInfo); + } + if (totalShardsAcrossSnapshots > maximumAllowedShardCount && request.indices().length == 0) { + String message = "Total shard count [" + + totalShardsAcrossSnapshots + + "] is more than the maximum allowed value of shard count [" + + maximumAllowedShardCount + + "] for snapshot status request"; + + throw new TooManyShardsInSnapshotsStatusException(repositoryName, message, request.snapshots()); + } + return unmodifiableMap(snapshotsInfoMap); + } + /** * Returns status of shards currently finished snapshots *

@@ -413,21 +476,65 @@ private SnapshotInfo snapshot(SnapshotsInProgress snapshotsInProgress, String re * {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it * returns similar information but for already finished snapshots. *

- * + * @param request snapshot status request * @param repositoryName repository name * @param snapshotInfo snapshot info * @return map of shard id to snapshot status */ private Map snapshotShards( + final SnapshotsStatusRequest request, final String repositoryName, final RepositoryData repositoryData, final SnapshotInfo snapshotInfo ) throws IOException { + final Set requestedIndexNames = Sets.newHashSet(request.indices()); + String snapshotName = snapshotInfo.snapshotId().getName(); + Set indices = Sets.newHashSet(snapshotInfo.indices()); + if (requestedIndexNames.isEmpty() == false) { + Set finalIndices = indices; + List indicesNotFound = requestedIndexNames.stream() + .filter(i -> finalIndices.contains(i) == false) + .collect(Collectors.toList()); + if (indicesNotFound.isEmpty() == false) { + handleIndexNotFound(String.join(", ", indicesNotFound), request, snapshotName, repositoryName); + } + indices = requestedIndexNames; + } + final Repository repository = repositoriesService.repository(repositoryName); - final Map shardStatus = new HashMap<>(); - for (String index : snapshotInfo.indices()) { + boolean isV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; + int totalShardsAcrossIndices = 0; + final Map indexMetadataMap = new HashMap<>(); + + for (String index : indices) { IndexId indexId = repositoryData.resolveIndexId(index); IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotInfo.snapshotId(), indexId); + if (indexMetadata != null) { + if (requestedIndexNames.isEmpty() == false && isV2Snapshot == false) { + totalShardsAcrossIndices += indexMetadata.getNumberOfShards(); + } + indexMetadataMap.put(indexId, indexMetadata); + } else if (requestedIndexNames.isEmpty() == false) { + handleIndexNotFound(index, request, snapshotName, repositoryName); + } + } + + if (totalShardsAcrossIndices > maximumAllowedShardCount && requestedIndexNames.isEmpty() == false && isV2Snapshot == false) { + String message = "Total shard count [" + + totalShardsAcrossIndices + + "] across the requested indices [" + + requestedIndexNames.stream().collect(Collectors.joining(", ")) + + "] is more than the maximum allowed value of shard count [" + + maximumAllowedShardCount + + "] for snapshot status request"; + + throw new TooManyShardsInSnapshotsStatusException(repositoryName, message, snapshotName); + } + + final Map shardStatus = new HashMap<>(); + for (Map.Entry entry : indexMetadataMap.entrySet()) { + IndexId indexId = entry.getKey(); + IndexMetadata indexMetadata = entry.getValue(); if (indexMetadata != null) { int numberOfShards = indexMetadata.getNumberOfShards(); for (int i = 0; i < numberOfShards; i++) { @@ -447,7 +554,12 @@ private Map snapshotShards( // could not be taken due to partial being set to false. shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped"); } else { - shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), indexId, shardId); + // TODO: to be refactored later + if (isV2Snapshot) { + shardSnapshotStatus = IndexShardSnapshotStatus.newDone(0, 0, 0, 0, 0, 0, null); + } else { + shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), indexId, shardId); + } } shardStatus.put(shardId, shardSnapshotStatus); } @@ -457,6 +569,21 @@ private Map snapshotShards( return unmodifiableMap(shardStatus); } + private void handleIndexNotFound(String index, SnapshotsStatusRequest request, String snapshotName, String repositoryName) { + if (request.ignoreUnavailable()) { + // ignoring unavailable index + logger.debug( + "snapshot status request ignoring indices [{}], not found in snapshot[{}] in repository [{}]", + index, + snapshotName, + repositoryName + ); + } else { + String cause = "indices [" + index + "] missing in snapshot [" + snapshotName + "] of repository [" + repositoryName + "]"; + throw new IndexNotFoundException(index, new IllegalArgumentException(cause)); + } + } + private static SnapshotShardFailure findShardFailure(List shardFailures, ShardId shardId) { for (SnapshotShardFailure shardFailure : shardFailures) { if (shardId.getIndexName().equals(shardFailure.index()) && shardId.getId() == shardFailure.shardId()) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 31e47dc14db23..72cc843d1b0e7 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -636,6 +636,7 @@ public void apply(Settings value, Settings current, Settings previous) { HandshakingTransportAddressConnector.PROBE_CONNECT_TIMEOUT_SETTING, HandshakingTransportAddressConnector.PROBE_HANDSHAKE_TIMEOUT_SETTING, SnapshotsService.MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, + SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API, FsHealthService.ENABLED_SETTING, FsHealthService.REFRESH_INTERVAL_SETTING, FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING, diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java index 2b4b188ab0acd..502be16f2fa8e 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java @@ -61,6 +61,7 @@ public class RestSnapshotsStatusAction extends BaseRestHandler { public List routes() { return unmodifiableList( asList( + new Route(GET, "/_snapshot/{repository}/{snapshot}/{index}/_status"), new Route(GET, "/_snapshot/{repository}/{snapshot}/_status"), new Route(GET, "/_snapshot/{repository}/_status"), new Route(GET, "/_snapshot/_status") @@ -80,7 +81,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (snapshots.length == 1 && "_all".equalsIgnoreCase(snapshots[0])) { snapshots = Strings.EMPTY_ARRAY; } - SnapshotsStatusRequest snapshotsStatusRequest = snapshotsStatusRequest(repository).snapshots(snapshots); + String[] indices = request.paramAsStringArray("index", Strings.EMPTY_ARRAY); + SnapshotsStatusRequest snapshotsStatusRequest = snapshotsStatusRequest(repository).snapshots(snapshots).indices(indices); snapshotsStatusRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", snapshotsStatusRequest.ignoreUnavailable())); snapshotsStatusRequest.clusterManagerNodeTimeout( diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 5ecb991127fd5..232dbe946d2ae 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -204,6 +204,18 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus Setting.Property.Dynamic ); + /** + * Setting to specify the maximum number of shards that can be included in the result for the snapshot status + * API call. Note that it does not apply to V2-shallow snapshots. + */ + public static final Setting MAX_SHARDS_ALLOWED_IN_STATUS_API = Setting.intSetting( + "snapshot.max_shards_allowed_in_status_api", + 200000, + 1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = ":"; private volatile int maxConcurrentOperations; diff --git a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java new file mode 100644 index 0000000000000..1689b3e4941ec --- /dev/null +++ b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.snapshots; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; + +import java.io.IOException; + +/** + * Thrown if the number of shards across the requested resources (snapshot(s) or the index/indices of a particular snapshot) + * breaches the limit of snapshot.max_shards_allowed_in_status_api cluster setting + * + * @opensearch.internal + */ +public class TooManyShardsInSnapshotsStatusException extends SnapshotException { + + public TooManyShardsInSnapshotsStatusException( + final String repositoryName, + final SnapshotId snapshotId, + final String message, + final Throwable cause + ) { + super(repositoryName, snapshotId, message, cause); + } + + public TooManyShardsInSnapshotsStatusException(final String repositoryName, final String message, String... snapshotName) { + super(repositoryName, String.join(", ", snapshotName), message); + } + + public TooManyShardsInSnapshotsStatusException(StreamInput in) throws IOException { + super(in); + } + + @Override + public RestStatus status() { + return RestStatus.REQUEST_ENTITY_TOO_LARGE; + } +} diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2e4a2d7bdd59c..eff312a36dbc6 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -119,6 +119,7 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotInUseDeletionException; +import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.transport.ActionNotFoundTransportException; @@ -898,6 +899,7 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); + ids.put(175, TooManyShardsInSnapshotsStatusException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); From bcd09ab65de2099152dd140668860491ca2c993c Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:50:23 +0530 Subject: [PATCH 013/357] Fix missing order field in TemplatesMetadata deserialization (#15509) Signed-off-by: Sooraj Sinha --- .../cluster/metadata/TemplatesMetadata.java | 15 ++++---- .../cluster/metadata/MetadataTests.java | 35 +++++++++++++++++++ .../model/RemoteTemplatesMetadataTests.java | 10 ++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 6ecc471c5e0ae..d31a51628eac4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -133,16 +133,15 @@ public static TemplatesMetadata fromXContent(XContentParser parser) throws IOExc String currentFieldName = parser.currentName(); if (currentFieldName == null) { token = parser.nextToken(); - if (token == XContentParser.Token.START_OBJECT) { - // move to the field name - token = parser.nextToken(); - } - currentFieldName = parser.currentName(); } - if (currentFieldName != null) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + // move to the field name + token = parser.nextToken(); + } + if (parser.currentName() != null && token != XContentParser.Token.END_OBJECT) { + do { builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); - } + } while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT); } return builder.build(); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java index a434a713f330b..c6cde72ab60b1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataTests.java @@ -1536,6 +1536,41 @@ public static Metadata randomMetadata() { return md.build(); } + public void testXContentWithTemplateMetadata() throws IOException { + final TemplatesMetadata templatesMetadata = getTemplatesMetadata(0); + verifyTemplatesMetadata(templatesMetadata); + final TemplatesMetadata templatesMetadata2 = getTemplatesMetadata(2); + verifyTemplatesMetadata(templatesMetadata2); + } + + private void verifyTemplatesMetadata(TemplatesMetadata templatesMetadata) throws IOException { + final Metadata originalMeta = Metadata.builder().templates(templatesMetadata).build(); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + Metadata.FORMAT.toXContent(builder, originalMeta); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final Metadata fromXContentMeta = Metadata.fromXContent(parser); + assertThat(fromXContentMeta.templates(), equalTo(originalMeta.templates())); + } + } + + private static TemplatesMetadata getTemplatesMetadata(int numberOfTemplates) { + TemplatesMetadata.Builder builder = TemplatesMetadata.builder(); + for (int i = 0; i < numberOfTemplates; i++) { + builder.put( + IndexTemplateMetadata.builder("template" + i) + .order(1234) + .patterns(Arrays.asList(randomAlphaOfLength(3) + "-*")) + .settings( + Settings.builder().put("index.random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build() + ) + .build() + ); + } + return builder.build(); + } + private static CreateIndexResult createIndices(int numIndices, int numBackingIndices, String dataStreamName) { // create some indices that do not back a data stream final List indices = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java index d7ecd2ad3f44a..ca978654c8825 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java @@ -231,12 +231,22 @@ public static TemplatesMetadata getTemplatesMetadata() { return TemplatesMetadata.builder() .put( IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .order(1234) .patterns(Arrays.asList("bar-*", "foo-*")) .settings( Settings.builder().put("index.random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build() ) .build() ) + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .order(5678) + .patterns(Arrays.asList("test-*")) + .settings( + Settings.builder().put("index.random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build() + ) + .build() + ) .build(); } } From 9f814797d0fe57457359a1f8a1b222cea80dc003 Mon Sep 17 00:00:00 2001 From: Somesh Gupta <35426854+aasom143@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:06:36 +0530 Subject: [PATCH 014/357] Support cancellation for admin apis (#13966) * Support cancellation for admin apis - add implementation for _cat/shards Signed-off-by: Somesh Gupta --- CHANGELOG.md | 1 + .../shards/TransportCatShardsActionIT.java | 129 ++++++++++++++++++ .../org/opensearch/action/ActionModule.java | 3 + .../admin/cluster/shards/CatShardsAction.java | 25 ++++ .../cluster/shards/CatShardsRequest.java | 62 +++++++++ .../cluster/shards/CatShardsResponse.java | 57 ++++++++ .../shards/TransportCatShardsAction.java | 110 +++++++++++++++ .../cluster/state/ClusterStateRequest.java | 13 ++ .../indices/stats/IndicesStatsRequest.java | 13 ++ .../indices/stats/IndicesStatsResponse.java | 2 +- .../action/search/TransportSearchAction.java | 5 +- .../TimeoutTaskCancellationUtility.java | 59 +++++--- .../support/broadcast/BroadcastRequest.java | 10 ++ .../ClusterManagerNodeReadRequest.java | 10 ++ .../admin/cluster/ClusterAdminTask.java | 47 +++++++ .../rest/action/cat/RestShardsAction.java | 36 ++--- .../shards/CatShardsResponseTests.java | 61 +++++++++ 17 files changed, 601 insertions(+), 42 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/cluster/ClusterAdminTask.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ff823efef1271..df0cb6113cd14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) - Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) +- Support cancellation for cat shards and node stats API.([#13966](https://github.com/opensearch-project/OpenSearch/pull/13966)) - [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381)) - Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) - Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java new file mode 100644 index 0000000000000..b86521dedf739 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java @@ -0,0 +1,129 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; +import static org.opensearch.common.unit.TimeValue.timeValueMillis; +import static org.opensearch.search.SearchService.NO_TIMEOUT; + +@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST) +public class TransportCatShardsActionIT extends OpenSearchIntegTestCase { + + public void testCatShardsWithSuccessResponse() throws InterruptedException { + internalCluster().startClusterManagerOnlyNodes(1); + List nodes = internalCluster().startDataOnlyNodes(3); + createIndex( + "test", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "60m") + .build() + ); + ensureGreen("test"); + + final CatShardsRequest shardsRequest = new CatShardsRequest(); + shardsRequest.setCancelAfterTimeInterval(NO_TIMEOUT); + shardsRequest.setIndices(Strings.EMPTY_ARRAY); + CountDownLatch latch = new CountDownLatch(1); + client().execute(CatShardsAction.INSTANCE, shardsRequest, new ActionListener() { + @Override + public void onResponse(CatShardsResponse catShardsResponse) { + ClusterStateResponse clusterStateResponse = catShardsResponse.getClusterStateResponse(); + IndicesStatsResponse indicesStatsResponse = catShardsResponse.getIndicesStatsResponse(); + for (ShardRouting shard : clusterStateResponse.getState().routingTable().allShards()) { + assertEquals("test", shard.getIndexName()); + assertNotNull(indicesStatsResponse.asMap().get(shard)); + } + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + fail(); + latch.countDown(); + } + }); + latch.await(); + } + + public void testCatShardsWithTimeoutException() throws IOException, AssertionError, InterruptedException { + List masterNodes = internalCluster().startClusterManagerOnlyNodes(1); + List nodes = internalCluster().startDataOnlyNodes(3); + createIndex( + "test", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2) + .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "60m") + .build() + ); + ensureGreen("test"); + + Settings clusterManagerDataPathSettings = internalCluster().dataPathSettings(masterNodes.get(0)); + // Dropping master node to delay in cluster state call. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0))); + + CountDownLatch latch = new CountDownLatch(2); + new Thread(() -> { + try { + // Ensures the cancellation timeout expires. + Thread.sleep(2000); + // Starting master node to proceed in cluster state call. + internalCluster().startClusterManagerOnlyNode( + Settings.builder().put("node.name", masterNodes.get(0)).put(clusterManagerDataPathSettings).build() + ); + latch.countDown(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }).start(); + + final CatShardsRequest shardsRequest = new CatShardsRequest(); + TimeValue timeoutInterval = timeValueMillis(1000); + shardsRequest.setCancelAfterTimeInterval(timeoutInterval); + shardsRequest.clusterManagerNodeTimeout(timeValueMillis(2500)); + shardsRequest.setIndices(Strings.EMPTY_ARRAY); + client().execute(CatShardsAction.INSTANCE, shardsRequest, new ActionListener() { + @Override + public void onResponse(CatShardsResponse catShardsResponse) { + // onResponse should not be called. + latch.countDown(); + throw new AssertionError( + "The cat shards action is expected to fail with a TaskCancelledException, but it received a successful response instead." + ); + } + + @Override + public void onFailure(Exception e) { + assertSame(e.getClass(), TaskCancelledException.class); + assertEquals(e.getMessage(), "Cancellation timeout of " + timeoutInterval + " is expired"); + latch.countDown(); + } + }); + latch.await(); + } + +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index c86e6580122d5..fbf90b97d1e8f 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -85,7 +85,9 @@ import org.opensearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsAction; import org.opensearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction; +import org.opensearch.action.admin.cluster.shards.CatShardsAction; import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsAction; +import org.opensearch.action.admin.cluster.shards.TransportCatShardsAction; import org.opensearch.action.admin.cluster.shards.TransportClusterSearchShardsAction; import org.opensearch.action.admin.cluster.shards.routing.weighted.delete.ClusterDeleteWeightedRoutingAction; import org.opensearch.action.admin.cluster.shards.routing.weighted.delete.TransportDeleteWeightedRoutingAction; @@ -646,6 +648,7 @@ public void reg actions.register(ClusterGetWeightedRoutingAction.INSTANCE, TransportGetWeightedRoutingAction.class); actions.register(ClusterDeleteWeightedRoutingAction.INSTANCE, TransportDeleteWeightedRoutingAction.class); actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class); + actions.register(CatShardsAction.INSTANCE, TransportCatShardsAction.class); actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); actions.register(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class); actions.register(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsAction.java new file mode 100644 index 0000000000000..381a950fe9a79 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsAction.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.action.ActionType; + +/** + * Transport action for cat shards + * + * @opensearch.internal + */ +public class CatShardsAction extends ActionType { + public static final CatShardsAction INSTANCE = new CatShardsAction(); + public static final String NAME = "cluster:monitor/shards"; + + private CatShardsAction() { + super(NAME, CatShardsResponse::new); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java new file mode 100644 index 0000000000000..49299777db8ae --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.rest.action.admin.cluster.ClusterAdminTask; + +import java.io.IOException; +import java.util.Map; + +/** + * A request of _cat/shards. + * + * @opensearch.api + */ +public class CatShardsRequest extends ClusterManagerNodeReadRequest { + + private String[] indices; + private TimeValue cancelAfterTimeInterval; + + public CatShardsRequest() {} + + public CatShardsRequest(StreamInput in) throws IOException { + super(in); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public void setIndices(String[] indices) { + this.indices = indices; + } + + public String[] getIndices() { + return this.indices; + } + + public void setCancelAfterTimeInterval(TimeValue timeout) { + this.cancelAfterTimeInterval = timeout; + } + + public TimeValue getCancelAfterTimeInterval() { + return this.cancelAfterTimeInterval; + } + + @Override + public ClusterAdminTask createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new ClusterAdminTask(id, type, action, parentTaskId, headers, this.cancelAfterTimeInterval); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java new file mode 100644 index 0000000000000..3dd88a2cda037 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * A response of a cat shards request. + * + * @opensearch.api + */ +public class CatShardsResponse extends ActionResponse { + + private ClusterStateResponse clusterStateResponse = null; + + private IndicesStatsResponse indicesStatsResponse = null; + + public CatShardsResponse() {} + + public CatShardsResponse(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + clusterStateResponse.writeTo(out); + indicesStatsResponse.writeTo(out); + } + + public void setClusterStateResponse(ClusterStateResponse clusterStateResponse) { + this.clusterStateResponse = clusterStateResponse; + } + + public ClusterStateResponse getClusterStateResponse() { + return this.clusterStateResponse; + } + + public void setIndicesStatsResponse(IndicesStatsResponse indicesStatsResponse) { + this.indicesStatsResponse = indicesStatsResponse; + } + + public IndicesStatsResponse getIndicesStatsResponse() { + return this.indicesStatsResponse; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java new file mode 100644 index 0000000000000..224d3cbc5f10a --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TimeoutTaskCancellationUtility; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.NotifyOnceListener; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +/** + * Perform cat shards action + * + * @opensearch.internal + */ +public class TransportCatShardsAction extends HandledTransportAction { + + private final NodeClient client; + + @Inject + public TransportCatShardsAction(NodeClient client, TransportService transportService, ActionFilters actionFilters) { + super(CatShardsAction.NAME, transportService, actionFilters, CatShardsRequest::new); + this.client = client; + } + + @Override + public void doExecute(Task parentTask, CatShardsRequest shardsRequest, ActionListener listener) { + final ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.setShouldCancelOnTimeout(true); + clusterStateRequest.local(shardsRequest.local()); + clusterStateRequest.clusterManagerNodeTimeout(shardsRequest.clusterManagerNodeTimeout()); + clusterStateRequest.clear().nodes(true).routingTable(true).indices(shardsRequest.getIndices()); + assert parentTask instanceof CancellableTask; + clusterStateRequest.setParentTask(client.getLocalNodeId(), parentTask.getId()); + + ActionListener originalListener = new NotifyOnceListener() { + @Override + protected void innerOnResponse(CatShardsResponse catShardsResponse) { + listener.onResponse(catShardsResponse); + } + + @Override + protected void innerOnFailure(Exception e) { + listener.onFailure(e); + } + }; + ActionListener cancellableListener = TimeoutTaskCancellationUtility.wrapWithCancellationListener( + client, + (CancellableTask) parentTask, + ((CancellableTask) parentTask).getCancellationTimeout(), + originalListener, + e -> { + originalListener.onFailure(e); + } + ); + CatShardsResponse catShardsResponse = new CatShardsResponse(); + try { + client.admin().cluster().state(clusterStateRequest, new ActionListener() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + catShardsResponse.setClusterStateResponse(clusterStateResponse); + IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); + indicesStatsRequest.setShouldCancelOnTimeout(true); + indicesStatsRequest.all(); + indicesStatsRequest.indices(shardsRequest.getIndices()); + indicesStatsRequest.setParentTask(client.getLocalNodeId(), parentTask.getId()); + try { + client.admin().indices().stats(indicesStatsRequest, new ActionListener() { + @Override + public void onResponse(IndicesStatsResponse indicesStatsResponse) { + catShardsResponse.setIndicesStatsResponse(indicesStatsResponse); + cancellableListener.onResponse(catShardsResponse); + } + + @Override + public void onFailure(Exception e) { + cancellableListener.onFailure(e); + } + }); + } catch (Exception e) { + cancellableListener.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + cancellableListener.onFailure(e); + } + }); + } catch (Exception e) { + cancellableListener.onFailure(e); + } + + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java index 90a52f7406d57..66bb64ca5580b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateRequest.java @@ -41,8 +41,12 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.rest.action.admin.cluster.ClusterAdminTask; +import org.opensearch.tasks.Task; import java.io.IOException; +import java.util.Map; /** * Transport request for obtaining cluster state @@ -211,4 +215,13 @@ public ClusterStateRequest waitForMetadataVersion(long waitForMetadataVersion) { this.waitForMetadataVersion = waitForMetadataVersion; return this; } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + if (this.getShouldCancelOnTimeout()) { + return new ClusterAdminTask(id, type, action, parentTaskId, headers); + } else { + return super.createTask(id, type, action, parentTaskId, headers); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java index 2b64464a76899..c36e53098d166 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -36,8 +36,12 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.rest.action.admin.cluster.ClusterAdminTask; +import org.opensearch.tasks.Task; import java.io.IOException; +import java.util.Map; /** * A request to get indices level stats. Allow to enable different stats to be returned. @@ -103,6 +107,15 @@ public IndicesStatsRequest groups(String... groups) { return this; } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + if (this.getShouldCancelOnTimeout()) { + return new ClusterAdminTask(id, type, action, parentTaskId, headers); + } else { + return super.createTask(id, type, action, parentTaskId, headers); + } + } + public String[] groups() { return this.flags.groups(); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java index 6242081cd2371..900a886481fe6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -69,7 +69,7 @@ public class IndicesStatsResponse extends BroadcastResponse { shards = in.readArray(ShardStats::new, (size) -> new ShardStats[size]); } - IndicesStatsResponse( + public IndicesStatsResponse( ShardStats[] shards, int totalShards, int successfulShards, diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 88bf7ebea8e52..8c4927afa9a14 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -312,8 +312,9 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< listener = TimeoutTaskCancellationUtility.wrapWithCancellationListener( client, (CancellableTask) task, - clusterService.getClusterSettings(), - listener + clusterService.getClusterSettings().get(SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING), + listener, + e -> {} ); } executeRequest(task, searchRequest, this::searchAsyncAction, listener); diff --git a/server/src/main/java/org/opensearch/action/support/TimeoutTaskCancellationUtility.java b/server/src/main/java/org/opensearch/action/support/TimeoutTaskCancellationUtility.java index a317a45eab31f..d2e7121a592cb 100644 --- a/server/src/main/java/org/opensearch/action/support/TimeoutTaskCancellationUtility.java +++ b/server/src/main/java/org/opensearch/action/support/TimeoutTaskCancellationUtility.java @@ -12,11 +12,12 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.opensearch.client.OriginSettingClient; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.core.tasks.TaskId; import org.opensearch.search.SearchService; import org.opensearch.tasks.CancellableTask; @@ -27,7 +28,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; -import static org.opensearch.action.search.TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING; /** * Utility to cancel a timeout task @@ -38,25 +38,25 @@ public class TimeoutTaskCancellationUtility { private static final Logger logger = LogManager.getLogger(TimeoutTaskCancellationUtility.class); + private static final AtomicBoolean executeResponseOrFailureOnce = new AtomicBoolean(true); + /** * Wraps a listener with a timeout listener {@link TimeoutRunnableListener} to schedule the task cancellation for provided tasks on * generic thread pool * @param client - {@link NodeClient} * @param taskToCancel - task to schedule cancellation for - * @param clusterSettings - {@link ClusterSettings} + * @param timeout - {@link TimeValue} * @param listener - original listener associated with the task * @return wrapped listener */ public static ActionListener wrapWithCancellationListener( NodeClient client, CancellableTask taskToCancel, - ClusterSettings clusterSettings, - ActionListener listener + TimeValue timeout, + ActionListener listener, + TimeoutHandler timeoutHandler ) { - final TimeValue globalTimeout = clusterSettings.get(SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING); - final TimeValue timeoutInterval = (taskToCancel.getCancellationTimeout() == null) - ? globalTimeout - : taskToCancel.getCancellationTimeout(); + final TimeValue timeoutInterval = (taskToCancel.getCancellationTimeout() == null) ? timeout : taskToCancel.getCancellationTimeout(); // Note: -1 (or no timeout) will help to turn off cancellation. The combinations will be request level set at -1 or request level // set to null and cluster level set to -1. ActionListener listenerToReturn = listener; @@ -72,24 +72,30 @@ public static ActionListener wrapWithCancellationListener( // force the origin to execute the cancellation as a system user new OriginSettingClient(client, TASKS_ORIGIN).admin() .cluster() - .cancelTasks( - cancelTasksRequest, - ActionListener.wrap( - r -> logger.debug( + .cancelTasks(cancelTasksRequest, new ActionListener() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + logger.debug( "Scheduled cancel task with timeout: {} for original task: {} is successfully completed", timeoutInterval, cancelTasksRequest.getTaskId() - ), - e -> logger.error( + ); + // Notify the timeoutHandler that the task was canceled due to timeout + timeoutHandler.onTimeout(new TaskCancelledException(cancelTasksRequest.getReason())); + } + + @Override + public void onFailure(Exception e) { + logger.error( new ParameterizedMessage( "Scheduled cancel task with timeout: {} for original task: {} is failed", timeoutInterval, cancelTasksRequest.getTaskId() ), e - ) - ) - ); + ); + } + }); }); wrappedListener.cancellable = client.threadPool().schedule(wrappedListener, timeoutInterval, ThreadPool.Names.GENERIC); listenerToReturn = wrappedListener; @@ -100,6 +106,22 @@ public static ActionListener wrapWithCancellationListener( return listenerToReturn; } + /** + * A functional interface used to handle the timeout of a cancellable task. + * Implementations of this interface provide a callback method that is invoked + * when a task is cancelled due to a timeout. + */ + public interface TimeoutHandler { + + /** + * Called when a cancellable task is cancelled due to a timeout. + * + * @param e the exception that contains details about the task cancellation, + * including the reason for cancellation. + */ + void onTimeout(TaskCancelledException e); + } + /** * Timeout listener which executes the provided runnable after timeout is expired and if a response/failure is not yet received. * If either a response/failure is received before timeout then the scheduled task is cancelled and response/failure is sent back to @@ -144,6 +166,7 @@ public void run() { if (executeRunnable.compareAndSet(true, false)) { timeoutRunnable.run(); } // else do nothing since either response/failure is already sent to client + } catch (Exception ex) { // ignore the exception logger.error( diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java index 8a27e032cec5e..784c43c938f2c 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/BroadcastRequest.java @@ -52,6 +52,8 @@ public class BroadcastRequest> extends protected String[] indices; private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + protected boolean shouldCancelOnTimeout = false; + public BroadcastRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); @@ -125,4 +127,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringArrayNullable(indices); indicesOptions.writeIndicesOptions(out); } + + public void setShouldCancelOnTimeout(boolean shouldCancelOnTimeout) { + this.shouldCancelOnTimeout = shouldCancelOnTimeout; + } + + public boolean getShouldCancelOnTimeout() { + return this.shouldCancelOnTimeout; + } } diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java index 6dcc6ed1b098e..c373107163082 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeReadRequest.java @@ -47,6 +47,8 @@ public abstract class ClusterManagerNodeReadRequest headers) { + this(id, type, action, parentTaskId, headers, NO_TIMEOUT); + } + + public ClusterAdminTask( + long id, + String type, + String action, + TaskId parentTaskId, + Map headers, + TimeValue cancelAfterTimeInterval + ) { + super(id, type, action, null, parentTaskId, headers, cancelAfterTimeInterval); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } +} diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index 2113b8d59c531..a7ad5fe6c14a3 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -32,10 +32,11 @@ package org.opensearch.rest.action.cat; -import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.shards.CatShardsAction; +import org.opensearch.action.admin.cluster.shards.CatShardsRequest; +import org.opensearch.action.admin.cluster.shards.CatShardsResponse; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.indices.stats.CommonStats; -import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.client.node.NodeClient; @@ -61,7 +62,6 @@ import org.opensearch.index.warmer.WarmerStats; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.action.RestActionListener; import org.opensearch.rest.action.RestResponseListener; import org.opensearch.search.suggest.completion.CompletionStats; @@ -73,6 +73,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.search.SearchService.NO_TIMEOUT; /** * _cat API action to get shard information @@ -107,25 +108,18 @@ protected void documentation(StringBuilder sb) { @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local())); - clusterStateRequest.clusterManagerNodeTimeout( - request.paramAsTime("cluster_manager_timeout", clusterStateRequest.clusterManagerNodeTimeout()) - ); - parseDeprecatedMasterTimeoutParameter(clusterStateRequest, request, deprecationLogger, getName()); - clusterStateRequest.clear().nodes(true).routingTable(true).indices(indices); - return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener(channel) { + final CatShardsRequest shardsRequest = new CatShardsRequest(); + shardsRequest.local(request.paramAsBoolean("local", shardsRequest.local())); + shardsRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", shardsRequest.clusterManagerNodeTimeout())); + shardsRequest.setCancelAfterTimeInterval(request.paramAsTime("cancel_after_time_interval", NO_TIMEOUT)); + shardsRequest.setIndices(indices); + parseDeprecatedMasterTimeoutParameter(shardsRequest, request, deprecationLogger, getName()); + return channel -> client.execute(CatShardsAction.INSTANCE, shardsRequest, new RestResponseListener(channel) { @Override - public void processResponse(final ClusterStateResponse clusterStateResponse) { - IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); - indicesStatsRequest.all(); - indicesStatsRequest.indices(indices); - client.admin().indices().stats(indicesStatsRequest, new RestResponseListener(channel) { - @Override - public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) throws Exception { - return RestTable.buildResponse(buildTable(request, clusterStateResponse, indicesStatsResponse), channel); - } - }); + public RestResponse buildResponse(CatShardsResponse catShardsResponse) throws Exception { + ClusterStateResponse clusterStateResponse = catShardsResponse.getClusterStateResponse(); + IndicesStatsResponse indicesStatsResponse = catShardsResponse.getIndicesStatsResponse(); + return RestTable.buildResponse(buildTable(request, clusterStateResponse, indicesStatsResponse), channel); } }); } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java new file mode 100644 index 0000000000000..d0b98f4d286ae --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.shards; + +import org.opensearch.action.admin.cluster.shards.CatShardsResponse; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.cluster.ClusterName; +import org.opensearch.test.OpenSearchTestCase; + +import static org.junit.Assert.assertEquals; + +public class CatShardsResponseTests extends OpenSearchTestCase { + + private final CatShardsResponse catShardsResponse = new CatShardsResponse(); + + public void testGetAndSetClusterStateResponse() { + ClusterName clusterName = new ClusterName("1"); + ClusterStateResponse clusterStateResponse = new ClusterStateResponse(clusterName, null, false); + catShardsResponse.setClusterStateResponse(clusterStateResponse); + + assertEquals(clusterStateResponse, catShardsResponse.getClusterStateResponse()); + } + + public void testGetAndSetIndicesStatsResponse() { + final IndicesStatsResponse indicesStatsResponse = new IndicesStatsResponse(null, 0, 0, 0, null); + catShardsResponse.setIndicesStatsResponse(indicesStatsResponse); + + assertEquals(indicesStatsResponse, catShardsResponse.getIndicesStatsResponse()); + } +} From b23c9de97e1380ba80e5b01b21729b553ab595a2 Mon Sep 17 00:00:00 2001 From: Sumit Bansal Date: Tue, 3 Sep 2024 12:29:00 +0530 Subject: [PATCH 015/357] Fix flaky test MasterServiceTests.testClusterStateUpdateLoggingWithDebugEnabled (#15204) * Fix test MasterServiceTests.testClusterStateUpdateLoggingWithDebugEnabled Signed-off-by: Sumit Bansal --- .../opensearch/cluster/service/MasterServiceTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index 7562dfc2e9d33..db9abe0310e40 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -534,8 +534,8 @@ public void onFailure(String source, Exception e) { }); assertBusy(mockAppender::assertAllExpectationsMatched); // verify stats values after state is published - assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess()); - assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed()); + assertBusy(() -> assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess())); + assertBusy(() -> assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed())); } } } @@ -699,8 +699,8 @@ public void onFailure(String source, Exception e) { }); assertBusy(mockAppender::assertAllExpectationsMatched); // verify stats values after state is published - assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess()); - assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed()); + assertBusy(() -> assertEquals(1, clusterManagerService.getClusterStateStats().getUpdateSuccess())); + assertBusy(() -> assertEquals(0, clusterManagerService.getClusterStateStats().getUpdateFailed())); } } } From 022acbaede71f7f9911ef2d2cafa456b921b3e92 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 3 Sep 2024 13:42:24 +0530 Subject: [PATCH 016/357] Disable pinned timestamp flag and mute testRemoteStoreCleanupForDeletedIndex (#15598) Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../java/org/opensearch/snapshots/DeleteSnapshotIT.java | 1 + .../main/java/org/opensearch/test/OpenSearchIntegTestCase.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 2331d52c3a1bc..41350a0f553f4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -285,6 +285,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9208") public void testRemoteStoreCleanupForDeletedIndex() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index f277c118c3d46..2bf610974767b 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2886,7 +2886,7 @@ private static Settings buildRemoteStoreNodeAttributes( } settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values())); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean()); - settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); + settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false); return settings.build(); } From a5590d1f3057c0be30765288b7efff040283d90b Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 3 Sep 2024 14:18:43 +0530 Subject: [PATCH 017/357] Adding awaitsFix to approximateRangeTest flaky tests (#15601) Signed-off-by: Bharathwaj G --- .../search/approximate/ApproximatePointRangeQueryTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index dd683d28f00f7..7fb75e43da6ec 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -112,6 +112,7 @@ protected String toString(int dimension, byte[] value) { } } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeUnderDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -150,6 +151,7 @@ protected String toString(int dimension, byte[] value) { } } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeOverDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -193,6 +195,7 @@ protected String toString(int dimension, byte[] value) { } } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeShortCircuit() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { From 0512e9ee74a3a8929bbaa3c415cca6ca49791585 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 3 Sep 2024 15:22:51 +0530 Subject: [PATCH 018/357] [Star tree] Star tree index validations (#15533) --------- Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 347 +++++++++++++++++- .../metadata/MetadataCreateIndexService.java | 70 ++++ .../MetadataIndexTemplateService.java | 2 + .../MetadataUpdateSettingsService.java | 7 + .../common/settings/ClusterSettings.java | 1 + .../common/settings/IndexScopedSettings.java | 2 + .../org/opensearch/index/IndexSettings.java | 9 +- .../CompositeIndexSettings.java | 15 +- .../startree/StarTreeIndexSettings.java | 21 ++ .../index/mapper/DocumentParser.java | 11 + .../index/mapper/MapperService.java | 14 + .../opensearch/index/mapper/ObjectMapper.java | 9 + .../index/mapper/StarTreeMapper.java | 68 +++- .../MetadataIndexTemplateServiceTests.java | 15 + .../TranslogFlushIntervalSettingsTests.java | 146 ++++++++ .../StarTreeDocValuesFormatTests.java | 21 +- .../index/mapper/ObjectMapperTests.java | 10 +- .../index/mapper/StarTreeMapperTests.java | 118 ++++++ 18 files changed, 860 insertions(+), 26 deletions(-) create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/TranslogFlushIntervalSettingsTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 52c6c6801a3c2..8cfb710679137 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -8,19 +8,28 @@ package org.opensearch.index.mapper; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesService; +import org.opensearch.search.SearchHit; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; import org.junit.Before; @@ -39,6 +48,10 @@ */ public class StarTreeMapperIT extends OpenSearchIntegTestCase { private static final String TEST_INDEX = "test"; + Settings settings = Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .build(); private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { try { @@ -116,6 +129,12 @@ private static XContentBuilder createMaxDimTestMapping() { .startObject() .field("name", "dim2") .endObject() + .startObject() + .field("name", "dim3") + .endObject() + .startObject() + .field("name", "dim4") + .endObject() .endArray() .endObject() .endObject() @@ -132,6 +151,10 @@ private static XContentBuilder createMaxDimTestMapping() { .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("dim4") + .field("type", "integer") + .field("doc_values", true) + .endObject() .endObject() .endObject(); } catch (IOException e) { @@ -223,6 +246,56 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea } } + private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, boolean isDuplicateMetric) { + XContentBuilder mapping = null; + try { + mapping = jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .endObject() + .startObject() + .field("name", "numeric_dv") + .endObject() + .startObject() + .field("name", isDuplicateDim ? "numeric_dv" : "numeric_dv1") // Duplicate dimension + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "numeric_dv") + .endObject() + .startObject() + .field("name", isDuplicateMetric ? "numeric_dv" : "numeric_dv1") // Duplicate metric + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric_dv1") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + fail("Failed to create mapping: " + e.getMessage()); + } + return mapping; + } + private static String getDim(boolean hasDocValues, boolean isKeyword) { if (hasDocValues) { return "numeric"; @@ -244,7 +317,7 @@ public final void setupNodeSettings() { } public void testValidCompositeIndex() { - prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); for (IndicesService service : dataNodeInstances) { final Index index = resolveIndex("test"); @@ -285,8 +358,91 @@ public void testValidCompositeIndex() { } } + public void testCompositeIndexWithIndexNotSpecified() { + Settings settings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .build(); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get() + ); + assertEquals( + "Failed to parse mapping [_doc]: Set 'index.composite_index' as true as part of index settings to use star tree index", + ex.getMessage() + ); + } + + public void testCompositeIndexWithHigherTranslogFlushSize() { + Settings settings = Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(513, ByteSizeUnit.MB)) + .build(); + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get() + ); + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '512mb' for composite index", ex.getMessage()); + } + + public void testCompositeIndexWithArraysInCompositeField() throws IOException { + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); + // Attempt to index a document with an array field + XContentBuilder doc = jsonBuilder().startObject() + .field("timestamp", "2023-06-01T12:00:00Z") + .startArray("numeric_dv") + .value(10) + .value(20) + .value(30) + .endArray() + .endObject(); + + // Index the document and refresh + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> client().prepareIndex(TEST_INDEX).setSource(doc).get() + ); + assertEquals( + "object mapping for [_doc] with array for [numeric_dv] cannot be accepted as field is also part of composite index mapping which does not accept arrays", + ex.getMessage() + ); + } + + public void testCompositeIndexWithArraysInNonCompositeField() throws IOException { + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); + // Attempt to index a document with an array field + XContentBuilder doc = jsonBuilder().startObject() + .field("timestamp", "2023-06-01T12:00:00Z") + .startArray("numeric") + .value(10) + .value(20) + .value(30) + .endArray() + .endObject(); + + // Index the document and refresh + IndexResponse indexResponse = client().prepareIndex(TEST_INDEX).setSource(doc).get(); + + assertEquals(RestStatus.CREATED, indexResponse.status()); + + client().admin().indices().prepareRefresh(TEST_INDEX).get(); + // Verify the document was indexed + SearchResponse searchResponse = client().prepareSearch(TEST_INDEX).setQuery(QueryBuilders.matchAllQuery()).get(); + + assertEquals(1, searchResponse.getHits().getTotalHits().value); + + // Verify the values in the indexed document + SearchHit hit = searchResponse.getHits().getAt(0); + assertEquals("2023-06-01T12:00:00Z", hit.getSourceAsMap().get("timestamp")); + + List values = (List) hit.getSourceAsMap().get("numeric"); + assertEquals(3, values.size()); + assertTrue(values.contains(10)); + assertTrue(values.contains(20)); + assertTrue(values.contains(30)); + } + public void testUpdateIndexWithAdditionOfStarTree() { - prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -296,7 +452,7 @@ public void testUpdateIndexWithAdditionOfStarTree() { } public void testUpdateIndexWithNewerStarTree() { - prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createTestMappingWithoutStarTree(false, false, false)).get(); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -309,7 +465,7 @@ public void testUpdateIndexWithNewerStarTree() { } public void testUpdateIndexWhenMappingIsDifferent() { - prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); // update some field in the mapping IllegalArgumentException ex = expectThrows( @@ -320,7 +476,7 @@ public void testUpdateIndexWhenMappingIsDifferent() { } public void testUpdateIndexWhenMappingIsSame() { - prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); // update some field in the mapping AcknowledgedResponse putMappingResponse = client().admin() @@ -368,7 +524,7 @@ public void testUpdateIndexWhenMappingIsSame() { public void testInvalidDimCompositeIndex() { IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(true, false, false)).get() + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(true, false, false)).get() ); assertEquals( "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field", @@ -379,8 +535,14 @@ public void testInvalidDimCompositeIndex() { public void testMaxDimsCompositeIndex() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) - .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2)) + () -> prepareCreate(TEST_INDEX).setSettings(settings) + .setMapping(createMaxDimTestMapping()) + .setSettings( + Settings.builder() + .put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + ) .get() ); assertEquals( @@ -389,11 +551,35 @@ public void testMaxDimsCompositeIndex() { ); } + public void testMaxMetricsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings) + .setMapping(createMaxDimTestMapping()) + .setSettings( + Settings.builder() + .put(StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING.getKey(), 4) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + ) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: There cannot be more than [4] base metrics for star tree field [startree-1]", + ex.getMessage() + ); + } + public void testMaxCalendarIntervalsCompositeIndex() { MapperParsingException ex = expectThrows( MapperParsingException.class, () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) - .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1)) + .setSettings( + Settings.builder() + .put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + ) .get() ); assertEquals( @@ -405,7 +591,7 @@ public void testMaxCalendarIntervalsCompositeIndex() { public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, true)).get() + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get() ); assertEquals( "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]", @@ -416,7 +602,7 @@ public void testUnsupportedDim() { public void testInvalidMetric() { IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, true, false)).get() + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, true, false)).get() ); assertEquals( "Aggregations not supported for the metrics field [numeric] with field type [integer] as part of star tree field", @@ -424,6 +610,145 @@ public void testInvalidMetric() { ); } + public void testDuplicateDimensions() { + XContentBuilder finalMapping = getMappingWithDuplicateFields(true, false); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(finalMapping).setSettings(settings).get() + ); + assertEquals( + "Failed to parse mapping [_doc]: Duplicate dimension [numeric_dv] present as part star tree index field [startree-1]", + ex.getMessage() + ); + } + + public void testDuplicateMetrics() { + XContentBuilder finalMapping = getMappingWithDuplicateFields(false, true); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(finalMapping).setSettings(settings).get() + ); + assertEquals( + "Failed to parse mapping [_doc]: Duplicate metrics [numeric_dv] present as part star tree index field [startree-1]", + ex.getMessage() + ); + } + + public void testValidTranslogFlushThresholdSize() { + Settings indexSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(256, ByteSizeUnit.MB)) + .build(); + + AcknowledgedResponse response = prepareCreate(TEST_INDEX).setSettings(indexSettings) + .setMapping(createMinimalTestMapping(false, false, false)) + .get(); + + assertTrue(response.isAcknowledged()); + } + + public void testInvalidTranslogFlushThresholdSize() { + Settings indexSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1024, ByteSizeUnit.MB)) + .build(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setSettings(indexSettings).setMapping(createMinimalTestMapping(false, false, false)).get() + ); + + assertTrue( + ex.getMessage().contains("You can configure 'index.translog.flush_threshold_size' with upto '512mb' for composite index") + ); + } + + public void testTranslogFlushThresholdSizeWithDefaultCompositeSettingLow() { + Settings updatedSettings = Settings.builder() + .put(CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "130m") + .build(); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(updatedSettings); + + client().admin().cluster().updateSettings(updateSettingsRequest).actionGet(); + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get() + ); + + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '130mb' for composite index", ex.getMessage()); + } + + public void testUpdateTranslogFlushThresholdSize() { + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get(); + + // Update to a valid value + AcknowledgedResponse validUpdateResponse = client().admin() + .indices() + .prepareUpdateSettings(TEST_INDEX) + .setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "256mb")) + .get(); + assertTrue(validUpdateResponse.isAcknowledged()); + + // Try to update to an invalid value + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> client().admin() + .indices() + .prepareUpdateSettings(TEST_INDEX) + .setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "1024mb")) + .get() + ); + + assertTrue( + ex.getMessage().contains("You can configure 'index.translog.flush_threshold_size' with upto '512mb' for composite index") + ); + + // update cluster settings to higher value + Settings updatedSettings = Settings.builder() + .put(CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "1030m") + .build(); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(updatedSettings); + + client().admin().cluster().updateSettings(updateSettingsRequest).actionGet(); + + // update index threshold flush to higher value + validUpdateResponse = client().admin() + .indices() + .prepareUpdateSettings(TEST_INDEX) + .setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "1024mb")) + .get(); + assertTrue(validUpdateResponse.isAcknowledged()); + } + + public void testMinimumTranslogFlushThresholdSize() { + Settings indexSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(56, ByteSizeUnit.BYTES)) + .build(); + + AcknowledgedResponse response = prepareCreate(TEST_INDEX).setSettings(indexSettings) + .setMapping(createMinimalTestMapping(false, false, false)) + .get(); + + assertTrue(response.isAcknowledged()); + } + + public void testBelowMinimumTranslogFlushThresholdSize() { + Settings indexSettings = Settings.builder() + .put(settings) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(55, ByteSizeUnit.BYTES)) + .build(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setSettings(indexSettings).setMapping(createMinimalTestMapping(false, false, false)).get() + ); + + assertEquals("failed to parse value [55b] for setting [index.translog.flush_threshold_size], must be >= [56b]", ex.getMessage()); + } + @After public final void cleanupNodeSettings() { assertAcked( diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index ad0ee0be59de2..8b08927bc146a 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -81,6 +81,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; @@ -88,7 +89,9 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.compositeindex.CompositeIndexValidator; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MapperService.MergeReason; @@ -153,6 +156,7 @@ import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplateName; import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; +import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; @@ -1072,6 +1076,7 @@ static Settings aggregateIndexSettings( validateTranslogRetentionSettings(indexSettings); validateStoreTypeSettings(indexSettings); validateRefreshIntervalSettings(request.settings(), clusterSettings); + validateTranslogFlushIntervalSettingsForCompositeIndex(request.settings(), clusterSettings); validateTranslogDurabilitySettings(request.settings(), clusterSettings, settings); return indexSettings; } @@ -1740,6 +1745,71 @@ public static void validateTranslogRetentionSettings(Settings indexSettings) { } } + /** + * Validates {@code index.translog.flush_threshold_size} is equal or below the {@code indices.composite_index.translog.max_flush_threshold_size} + * for composite indices based on {{@code index.composite_index}} + * + * @param requestSettings settings passed in during index create/update request + * @param clusterSettings cluster setting + */ + public static void validateTranslogFlushIntervalSettingsForCompositeIndex(Settings requestSettings, ClusterSettings clusterSettings) { + if (StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.exists(requestSettings) == false + || requestSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey()) == null) { + return; + } + ByteSizeValue translogFlushSize = INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(requestSettings); + ByteSizeValue compositeIndexMaxFlushSize = clusterSettings.get( + CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING + ); + if (translogFlushSize.compareTo(compositeIndexMaxFlushSize) > 0) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "You can configure '%s' with upto '%s' for composite index", + INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), + compositeIndexMaxFlushSize + ) + ); + } + } + + /** + * Validates {@code index.translog.flush_threshold_size} is equal or below the {@code indices.composite_index.translog.max_flush_threshold_size} + * for composite indices based on {{@code index.composite_index}} + * This is used during update index settings flow + * + * @param requestSettings settings passed in during index update request + * @param clusterSettings cluster setting + * @param indexSettings index settings + */ + public static Optional validateTranslogFlushIntervalSettingsForCompositeIndex( + Settings requestSettings, + ClusterSettings clusterSettings, + Settings indexSettings + ) { + if (INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.exists(requestSettings) == false + || requestSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()) == null + || StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.exists(indexSettings) == false + || indexSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey()) == null) { + return Optional.empty(); + } + ByteSizeValue translogFlushSize = INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(requestSettings); + ByteSizeValue compositeIndexMaxFlushSize = clusterSettings.get( + CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING + ); + if (translogFlushSize.compareTo(compositeIndexMaxFlushSize) > 0) { + return Optional.of( + String.format( + Locale.ROOT, + "You can configure '%s' with upto '%s' for composite index", + INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), + compositeIndexMaxFlushSize + ) + ); + } + return Optional.empty(); + } + /** * Validates {@code index.refresh_interval} is equal or below the {@code cluster.minimum.index.refresh_interval}. * diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index a7b9eba6dbc05..e4afc798cc64d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -102,6 +102,7 @@ import static org.opensearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings; +import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex; import static org.opensearch.common.util.concurrent.ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME; import static org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -1639,6 +1640,7 @@ private void validate(String name, @Nullable Settings settings, List ind // validate index refresh interval and translog durability settings validateRefreshIntervalSettings(settings, clusterService.getClusterSettings()); + validateTranslogFlushIntervalSettingsForCompositeIndex(settings, clusterService.getClusterSettings()); validateTranslogDurabilitySettingsInTemplate(settings, clusterService.getClusterSettings()); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index 6713b85fee25a..4e7e31bbb9222 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -82,6 +82,7 @@ import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateOverlap; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogDurabilitySettings; +import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex; import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findComponentTemplate; import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX; import static org.opensearch.index.IndexSettings.same; @@ -221,6 +222,12 @@ public ClusterState execute(ClusterState currentState) { index.getName() ).ifPresent(validationErrors::add); } + validateTranslogFlushIntervalSettingsForCompositeIndex( + normalizedSettings, + clusterService.getClusterSettings(), + metadata.getSettings() + ).ifPresent(validationErrors::add); + } if (validationErrors.size() > 0) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 72cc843d1b0e7..e5f018aca5c0a 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -771,6 +771,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Composite index settings CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING, + CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 9cad80e8e3e82..00c73de1c1780 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -249,6 +249,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { StarTreeIndexSettings.DEFAULT_METRICS_LIST, StarTreeIndexSettings.DEFAULT_DATE_INTERVALS, StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING, + StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING, + StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING, IndexSettings.INDEX_CONTEXT_CREATED_VERSION, IndexSettings.INDEX_CONTEXT_CURRENT_VERSION, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 12715b353c526..3e58b9df63378 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,6 +48,7 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.Index; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; @@ -889,6 +890,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { */ private volatile double docIdFuzzySetFalsePositiveProbability; + private final boolean isCompositeIndex; + /** * Returns the default search fields for this index. */ @@ -1052,7 +1055,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti setEnableFuzzySetForDocId(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING)); setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING)); - + isCompositeIndex = scopedSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING); scopedSettings.addSettingsUpdateConsumer( TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING, tieredMergePolicyProvider::setNoCFSRatio @@ -1297,6 +1300,10 @@ public int getNumberOfReplicas() { return settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null); } + public boolean isCompositeIndex() { + return isCompositeIndex; + } + /** * Returns true if segment replication is enabled on the index. * diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java index 014dd22426a10..a29e642d30f05 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java @@ -13,6 +13,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; /** * Cluster level settings for composite indices @@ -37,12 +39,23 @@ public class CompositeIndexSettings { Setting.Property.Dynamic ); + /** + * This sets the max flush threshold size for composite index + */ + public static final Setting COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING = Setting.byteSizeSetting( + "indices.composite_index.translog.max_flush_threshold_size", + new ByteSizeValue(512, ByteSizeUnit.MB), + new ByteSizeValue(128, ByteSizeUnit.MB), + new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private volatile boolean starTreeIndexCreationEnabled; public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) { this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled); - } private void starTreeIndexCreationEnabled(boolean value) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index ce389a99b3626..e665831b83d93 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -26,6 +26,7 @@ public class StarTreeIndexSettings { public static int STAR_TREE_MAX_DIMENSIONS_DEFAULT = 10; + public static int STAR_TREE_MAX_BASE_METRICS_DEFAULT = 100; /** * This setting determines the max number of star tree fields that can be part of composite index mapping. For each * star tree field, we will generate associated star tree index. @@ -52,6 +53,19 @@ public class StarTreeIndexSettings { Setting.Property.Final ); + /** + * This setting determines the max number of dimensions that can be part of star tree index field. Number of + * dimensions and associated cardinality has direct effect of star tree index size and query performance. + */ + public static final Setting STAR_TREE_MAX_BASE_METRICS_SETTING = Setting.intSetting( + "index.composite_index.star_tree.field.max_base_metrics", + STAR_TREE_MAX_BASE_METRICS_DEFAULT, + 4, + 100, + Setting.Property.IndexScope, + Setting.Property.Final + ); + /** * This setting determines the max number of date intervals that can be part of star tree date field. */ @@ -108,4 +122,11 @@ public static Rounding.DateTimeUnit getTimeUnit(String expression) { } return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); } + + public static final Setting IS_COMPOSITE_INDEX_SETTING = Setting.boolSetting( + "index.composite_index", + false, + Setting.Property.IndexScope, + Setting.Property.Final + ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index b03026d560dbf..50ff816695156 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -661,6 +661,17 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp throws IOException { XContentParser parser = context.parser(); XContentParser.Token token; + // block array values for composite index fields + if (context.indexSettings().isCompositeIndex() && context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)) { + throw new MapperParsingException( + String.format( + Locale.ROOT, + "object mapping for [%s] with array for [%s] cannot be accepted as field is also part of composite index mapping which does not accept arrays", + mapper.name(), + arrayFieldName + ) + ); + } final String[] paths = splitAndValidatePath(lastFieldName); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 400232ff164db..f75aa0d367ff7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -227,6 +227,7 @@ public enum MergeReason { private final BooleanSupplier idFieldDataEnabled; private volatile Set compositeMappedFieldTypes; + private volatile Set fieldsPartOfCompositeMappings; public MapperService( IndexSettings indexSettings, @@ -547,9 +548,18 @@ private synchronized Map internalMerge(DocumentMapper ma // initialize composite fields post merge this.compositeMappedFieldTypes = getCompositeFieldTypesFromMapper(); + buildCompositeFieldLookup(); return results; } + private void buildCompositeFieldLookup() { + Set fieldsPartOfCompositeMappings = new HashSet<>(); + for (CompositeMappedFieldType fieldType : compositeMappedFieldTypes) { + fieldsPartOfCompositeMappings.addAll(fieldType.fields()); + } + this.fieldsPartOfCompositeMappings = fieldsPartOfCompositeMappings; + } + private boolean assertSerialization(DocumentMapper mapper) { // capture the source now, it may change due to concurrent parsing final CompressedXContent mappingSource = mapper.mappingSource(); @@ -677,6 +687,10 @@ private Set getCompositeFieldTypesFromMapper() { return compositeMappedFieldTypes; } + public boolean isFieldPartOfCompositeIndex(String field) { + return fieldsPartOfCompositeMappings.contains(field); + } + public ObjectMapper getObjectMapper(String name) { return this.mapper == null ? null : this.mapper.objectMappers().get(name); } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 533e6ca73d737..dd984373fc9df 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -440,6 +440,15 @@ protected static void parseCompositeField( + " feature flag in the JVM options" ); } + if (StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.get(parserContext.getSettings()) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Set '%s' as true as part of index settings to use star tree index", + StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey() + ) + ); + } Iterator> iterator = compositeNode.entrySet().iterator(); if (compositeNode.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index e52d6a621e4e8..17c27ef149e54 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -22,6 +22,7 @@ import org.opensearch.search.lookup.SearchLookup; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; @@ -155,8 +156,20 @@ private List buildDimensions(String fieldName, Map ma String.format(Locale.ROOT, "Atleast two dimensions are required to build star tree index field [%s]", fieldName) ); } + Set dimensionFieldNames = new HashSet<>(); for (Object dim : dimList) { - dimensions.add(getDimension(fieldName, dim, context)); + Dimension dimension = getDimension(fieldName, dim, context); + if (dimensionFieldNames.add(dimension.getField()) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Duplicate dimension [%s] present as part star tree index field [%s]", + dimension.getField(), + fieldName + ) + ); + } + dimensions.add(dimension); } } else { throw new MapperParsingException( @@ -223,6 +236,7 @@ private List buildMetrics(String fieldName, Map map, Map } if (metricsFromInput instanceof List) { List metricsList = (List) metricsFromInput; + Set metricFieldNames = new HashSet<>(); for (Object metric : metricsList) { Map metricMap = (Map) metric; String name = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.NAME, metricMap); @@ -232,7 +246,18 @@ private List buildMetrics(String fieldName, Map map, Map } metricMap.remove(CompositeDataCubeFieldType.NAME); if (objbuilder == null || objbuilder.mappersBuilders == null) { - metrics.add(getMetric(name, metricMap, context)); + Metric metricFromParser = getMetric(name, metricMap, context); + if (metricFieldNames.add(metricFromParser.getField()) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Duplicate metrics [%s] present as part star tree index field [%s]", + metricFromParser.getField(), + fieldName + ) + ); + } + metrics.add(metricFromParser); } else { Optional meticBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); if (meticBuilder.isEmpty()) { @@ -243,7 +268,18 @@ private List buildMetrics(String fieldName, Map map, Map String.format(Locale.ROOT, "non-numeric field type is associated with star tree metric [%s]", this.name) ); } - metrics.add(getMetric(name, metricMap, context)); + Metric metricFromParser = getMetric(name, metricMap, context); + if (metricFieldNames.add(metricFromParser.getField()) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Duplicate metrics [%s] present as part star tree index field [%s]", + metricFromParser.getField(), + fieldName + ) + ); + } + metrics.add(metricFromParser); DocumentMapperParser.checkNoRemainingFields( metricMap, context.indexVersionCreated(), @@ -254,6 +290,32 @@ private List buildMetrics(String fieldName, Map map, Map } else { throw new MapperParsingException(String.format(Locale.ROOT, "unable to parse metrics for star tree field [%s]", this.name)); } + int numBaseMetrics = 0; + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getMetrics()) { + if (metricStat.isDerivedMetric() == false) { + numBaseMetrics++; + } + } + } + if (numBaseMetrics > context.getSettings() + .getAsInt( + StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING.getKey(), + StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_DEFAULT + )) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "There cannot be more than [%s] base metrics for star tree field [%s]", + context.getSettings() + .getAsInt( + StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING.getKey(), + StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_DEFAULT + ), + fieldName + ) + ); + } Metric docCountMetric = new Metric(DocCountFieldMapper.NAME, List.of(MetricStat.DOC_COUNT)); metrics.add(docCountMetric); return metrics; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 8312267205d62..99e259c8170f3 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -59,6 +59,8 @@ import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; +import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; @@ -2425,6 +2427,19 @@ public void testAsyncTranslogDurabilityBlocked() { assertThat(throwables.get(0), instanceOf(IllegalArgumentException.class)); } + public void testMaxTranslogFlushSizeWithCompositeIndex() { + Settings clusterSettings = Settings.builder() + .put(CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "130m") + .build(); + PutRequest request = new PutRequest("test", "test_replicas"); + request.patterns(singletonList("test_shards_wait*")); + Settings.Builder settingsBuilder = builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), "true") + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "131m"); + request.settings(settingsBuilder.build()); + List throwables = putTemplate(xContentRegistry(), request, clusterSettings); + assertThat(throwables.get(0), instanceOf(IllegalArgumentException.class)); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { return putTemplate(xContentRegistry, request, Settings.EMPTY); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/TranslogFlushIntervalSettingsTests.java b/server/src/test/java/org/opensearch/cluster/metadata/TranslogFlushIntervalSettingsTests.java new file mode 100644 index 0000000000000..f54b6cdf1b152 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/TranslogFlushIntervalSettingsTests.java @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Optional; + +/** + * Tests for translog flush interval settings update with and without composite index + */ +public class TranslogFlushIntervalSettingsTests extends OpenSearchTestCase { + + Settings settings = Settings.builder() + .put(CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "130mb") + .build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + public void testValidSettings() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "50mb") + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .build(); + + // This should not throw an exception + MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings); + } + + public void testDefaultTranslogFlushSetting() { + Settings requestSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build(); + + // This should not throw an exception + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings) + ); + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '130mb' for composite index", ex.getMessage()); + } + + public void testMissingCompositeIndexSetting() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "50mb") + .build(); + + // This should not throw an exception + MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings); + } + + public void testNullTranslogFlushSetting() { + Settings requestSettings = Settings.builder() + .putNull(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .build(); + + // This should not throw an exception + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings) + ); + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '130mb' for composite index", ex.getMessage()); + } + + public void testExceedingMaxFlushSize() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "150mb") + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .build(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings) + ); + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '130mb' for composite index", ex.getMessage()); + } + + public void testEqualToMaxFlushSize() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "100mb") + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .build(); + + // This should not throw an exception + MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings); + } + + public void testUpdateIndexThresholdFlushSize() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "100mb") + .build(); + + Settings indexSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build(); + + // This should not throw an exception + assertTrue( + MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex( + requestSettings, + clusterSettings, + indexSettings + ).isEmpty() + ); + } + + public void testUpdateFlushSizeAboveThresholdWithCompositeIndex() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "131mb") + .build(); + + Settings indexSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build(); + + Optional err = MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex( + requestSettings, + clusterSettings, + indexSettings + ); + assertTrue(err.isPresent()); + assertEquals("You can configure 'index.translog.flush_threshold_size' with upto '130mb' for composite index", err.get()); + } + + public void testUpdateFlushSizeAboveThresholdWithoutCompositeIndex() { + Settings requestSettings = Settings.builder() + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "131mb") + .build(); + + Settings indexSettings = Settings.builder().build(); + + // This should not throw an exception + assertTrue( + MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex( + requestSettings, + clusterSettings, + indexSettings + ).isEmpty() + ); + } +} diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index fa492e1adec0a..4bbefeba0845b 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -33,14 +33,18 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexSettings; import org.opensearch.index.MapperTestUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; import org.opensearch.index.codec.composite.composite99.Composite99Codec; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -213,20 +217,19 @@ private XContentBuilder topMapping(CheckedConsumer } private void createMapperService(XContentBuilder builder) throws IOException { - IndexMetadata indexMetadata = IndexMetadata.builder("test") - .settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - ) - .putMapping(builder.toString()) + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build(); IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), createTempDir(), - Settings.EMPTY, + settings, indicesModule, "test" ); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index 504bc622ec12e..ff0533de4fe8d 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -37,7 +37,11 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.index.mapper.ObjectMapper.Dynamic; import org.opensearch.plugins.Plugin; @@ -544,7 +548,11 @@ public void testCompositeFields() throws Exception { final Settings starTreeEnabledSettings = Settings.builder().put(STAR_TREE_INDEX, "true").build(); FeatureFlags.initializeFeatureFlags(starTreeEnabledSettings); - DocumentMapper documentMapper = createIndex("test").mapperService() + Settings settings = Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .build(); + DocumentMapper documentMapper = createIndex("test", settings).mapperService() .documentMapperParser() .parse("tweet", new CompressedXContent(mapping)); diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 81454b210d6be..e06d9889ec905 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -13,6 +13,8 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.compositeindex.CompositeIndexValidator; @@ -24,6 +26,7 @@ import org.opensearch.index.compositeindex.datacube.ReadDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.junit.After; import org.junit.Before; @@ -35,6 +38,9 @@ import java.util.List; import java.util.Set; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING; +import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING; import static org.hamcrest.Matchers.containsString; /** @@ -52,7 +58,17 @@ public void teardown() { FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } + @Override + protected Settings getIndexSettings() { + return Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .put(SETTINGS) + .build(); + } + public void testValidStarTree() throws IOException { + MapperService mapperService = createMapperService(getExpandedMappingWithJustAvg("status", "size")); Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { @@ -82,6 +98,40 @@ public void testValidStarTree() throws IOException { } } + public void testCompositeIndexWithArraysInCompositeField() throws IOException { + DocumentMapper mapper = createDocumentMapper(getExpandedMappingWithJustAvg("status", "status")); + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> mapper.parse(source(b -> b.startArray("status").value(0).value(1).endArray())) + ); + assertEquals( + "object mapping for [_doc] with array for [status] cannot be accepted as field is also part of composite index mapping which does not accept arrays", + ex.getMessage() + ); + ParsedDocument doc = mapper.parse(source(b -> b.startArray("size").value(0).value(1).endArray())); + // 1 intPoint , 1 SNDV field for each value , so 4 in total + assertEquals(4, doc.rootDoc().getFields("size").length); + } + + public void testValidValueForFlushTresholdSizeWithoutCompositeIndex() { + Settings settings = Settings.builder() + .put(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "256mb") + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), false) + .build(); + + assertEquals(new ByteSizeValue(256, ByteSizeUnit.MB), INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(settings)); + } + + public void testValidValueForCompositeIndex() { + Settings settings = Settings.builder() + .put(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "256mb") + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "512mb") + .build(); + + assertEquals(new ByteSizeValue(256, ByteSizeUnit.MB), INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(settings)); + } + public void testMetricsWithJustSum() throws IOException { MapperService mapperService = createMapperService(getExpandedMappingWithJustSum("status", "size")); Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); @@ -291,6 +341,24 @@ public void testInvalidSingleDim() { ); } + public void testDuplicateDimensions() { + XContentBuilder finalMapping = getMappingWithDuplicateFields(true, false); + MapperParsingException ex = expectThrows(MapperParsingException.class, () -> createMapperService(finalMapping)); + assertEquals( + "Failed to parse mapping [_doc]: Duplicate dimension [numeric_dv] present as part star tree index field [startree-1]", + ex.getMessage() + ); + } + + public void testDuplicateMetrics() { + XContentBuilder finalMapping = getMappingWithDuplicateFields(false, true); + MapperParsingException ex = expectThrows(MapperParsingException.class, () -> createMapperService(finalMapping)); + assertEquals( + "Failed to parse mapping [_doc]: Duplicate metrics [numeric_dv] present as part star tree index field [startree-1]", + ex.getMessage() + ); + } + public void testMetric() { List m1 = new ArrayList<>(); m1.add(MetricStat.MAX); @@ -507,6 +575,56 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) }); } + private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, boolean isDuplicateMetric) { + XContentBuilder mapping = null; + try { + mapping = jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .endObject() + .startObject() + .field("name", "numeric_dv") + .endObject() + .startObject() + .field("name", isDuplicateDim ? "numeric_dv" : "numeric_dv1") // Duplicate dimension + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "numeric_dv") + .endObject() + .startObject() + .field("name", isDuplicateMetric ? "numeric_dv" : "numeric_dv1") // Duplicate metric + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric_dv1") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + fail("Failed to create mapping: " + e.getMessage()); + } + return mapping; + } + private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) throws IOException { return topMapping(b -> { b.startObject("composite"); From 33be5a9d161e496517842ded0bc4b04ecca674ff Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Tue, 3 Sep 2024 15:33:21 +0530 Subject: [PATCH 019/357] Add prefix support to hashed prefix & infix path types on remote store (#15557) * Add prefix support to hashed prefix & infix path types on remote store Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../RemotePrimaryLocalRecoveryIT.java | 5 +- .../remotestore/RemoteRestoreSnapshotIT.java | 10 +- .../opensearch/remotestore/RemoteStoreIT.java | 60 +++++- .../RemoteStoreRefreshListenerIT.java | 6 +- .../snapshots/DeleteSnapshotIT.java | 4 +- .../TransportCleanupRepositoryAction.java | 9 +- .../common/settings/ClusterSettings.java | 2 + .../index/remote/RemoteIndexPath.java | 11 +- .../index/remote/RemoteIndexPathUploader.java | 9 +- .../index/remote/RemoteStoreEnums.java | 12 +- .../index/remote/RemoteStorePathStrategy.java | 12 ++ .../opensearch/index/shard/IndexShard.java | 9 +- .../opensearch/index/shard/StoreRecovery.java | 4 +- .../RemoteSegmentStoreDirectoryFactory.java | 13 +- .../RemoteStoreLockManagerFactory.java | 10 +- .../index/translog/RemoteFsTranslog.java | 2 + .../indices/RemoteStoreSettings.java | 33 ++++ .../main/java/org/opensearch/node/Node.java | 12 +- .../blobstore/BlobStoreRepository.java | 7 +- .../snapshots/SnapshotsService.java | 9 +- .../GatewayMetaStatePersistedStateTests.java | 11 +- .../RemoteClusterStateServiceTests.java | 61 +++++- .../opensearch/index/IndexModuleTests.java | 2 +- .../index/remote/RemoteIndexPathTests.java | 106 ++++++++++- .../remote/RemoteIndexPathUploaderTests.java | 22 ++- ...emoteStoreCustomMetadataResolverTests.java | 46 +++++ .../index/remote/RemoteStoreEnumsTests.java | 179 ++++++++++++++---- ...moteSegmentStoreDirectoryFactoryTests.java | 2 +- .../RemoteStoreLockManagerFactoryTests.java | 2 +- .../RepositoriesServiceTests.java | 5 + .../BlobStoreRepositoryHelperTests.java | 6 +- .../snapshots/SnapshotResiliencyTests.java | 8 +- .../blobstore/BlobStoreTestUtil.java | 5 + .../AbstractSnapshotIntegTestCase.java | 5 +- .../test/OpenSearchIntegTestCase.java | 14 ++ .../opensearch/test/OpenSearchTestCase.java | 4 +- 37 files changed, 617 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df0cb6113cd14..eeb5f952b185f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) +- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java index 024fc68602a19..8ab093b8495cb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java @@ -19,6 +19,7 @@ import org.opensearch.core.util.FileSystemUtils; import org.opensearch.index.remote.RemoteSegmentStats; import org.opensearch.index.translog.RemoteTranslogStats; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -67,6 +68,7 @@ public void testLocalRecoveryRollingRestartAndNodeFailure() throws Exception { assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0); } + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); assertBusy(() -> { String shardPath = getShardLevelBlobPath( client(), @@ -74,7 +76,8 @@ public void testLocalRecoveryRollingRestartAndNodeFailure() throws Exception { new BlobPath(), String.valueOf(shardRouting.getId()), SEGMENTS, - DATA + DATA, + segmentsPathFixedPrefix ).buildAsString(); Path segmentDataRepoPath = segmentRepoPath.resolve(shardPath); List segmentsNFilesInRepo = Arrays.stream(FileSystemUtils.files(segmentDataRepoPath)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 56078a6ef8800..0acb578e2e7bf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -443,13 +443,15 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException { Client client = client(); - String path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, METADATA).buildAsString(); + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, METADATA, translogPathFixedPrefix).buildAsString(); Path remoteTranslogMetadataPath = Path.of(remoteRepoPath + "/" + path); - path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, DATA).buildAsString(); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, DATA, translogPathFixedPrefix).buildAsString(); Path remoteTranslogDataPath = Path.of(remoteRepoPath + "/" + path); - path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, METADATA).buildAsString(); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, METADATA, segmentsPathFixedPrefix).buildAsString(); Path segmentMetadataPath = Path.of(remoteRepoPath + "/" + path); - path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, DATA).buildAsString(); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, DATA, segmentsPathFixedPrefix).buildAsString(); Path segmentDataPath = Path.of(remoteRepoPath + "/" + path); try ( diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 9a8d3651160c0..692727357a88a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -205,7 +205,16 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString(); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String shardPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + SEGMENTS, + METADATA, + segmentsPathFixedPrefix + ).buildAsString(); Path indexPath = Path.of(segmentRepoPath + "/" + shardPath); ; IndexShard indexShard = getIndexShard(dataNode, INDEX_NAME); @@ -236,7 +245,16 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, false, INDEX_NAME); - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString(); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String shardPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + SEGMENTS, + METADATA, + segmentsPathFixedPrefix + ).buildAsString(); Path indexPath = Path.of(segmentRepoPath + "/" + shardPath); int actualFileCount = getFileCount(indexPath); // We also allow (numberOfIterations + 1) as index creation also triggers refresh. @@ -247,11 +265,19 @@ public void testStaleCommitDeletionWithMinSegmentFiles_3() throws Exception { Settings.Builder settings = Settings.builder() .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), "3"); internalCluster().startNode(settings); - + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString(); + String shardPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + SEGMENTS, + METADATA, + segmentsPathFixedPrefix + ).buildAsString(); Path indexPath = Path.of(segmentRepoPath + "/" + shardPath); int actualFileCount = getFileCount(indexPath); // We also allow (numberOfIterations + 1) as index creation also triggers refresh. @@ -271,7 +297,16 @@ public void testStaleCommitDeletionWithMinSegmentFiles_Disabled() throws Excepti createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(12, 18); indexData(numberOfIterations, true, INDEX_NAME); - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString(); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String shardPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + SEGMENTS, + METADATA, + segmentsPathFixedPrefix + ).buildAsString(); Path indexPath = Path.of(segmentRepoPath + "/" + shardPath); ; int actualFileCount = getFileCount(indexPath); @@ -604,8 +639,10 @@ public void testFallbackToNodeToNodeSegmentCopy() throws Exception { indexBulk(INDEX_NAME, 50); flushAndRefresh(INDEX_NAME); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); // 3. Delete data from remote segment store - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, DATA).buildAsString(); + String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, DATA, segmentsPathFixedPrefix) + .buildAsString(); Path segmentDataPath = Path.of(segmentRepoPath + "/" + shardPath); try (Stream files = Files.list(segmentDataPath)) { @@ -844,7 +881,16 @@ public void testLocalOnlyTranslogCleanupOnNodeRestart() throws Exception { .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); - String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", TRANSLOG, METADATA).buildAsString(); + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); Path translogMetaDataPath = Path.of(translogRepoPath + "/" + shardPath); try (Stream files = Files.list(translogMetaDataPath)) { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java index 7ae08bf968ade..a7292482100e1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java @@ -13,6 +13,7 @@ import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; @@ -50,7 +51,10 @@ public void testRemoteRefreshRetryOnFailure() throws Exception { String indexName = response.getShards()[0].getShardRouting().index().getName(); String indexUuid = response.getShards()[0].getShardRouting().index().getUUID(); - String shardPath = getShardLevelBlobPath(client(), indexName, new BlobPath(), "0", SEGMENTS, DATA).buildAsString(); + + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String shardPath = getShardLevelBlobPath(client(), indexName, new BlobPath(), "0", SEGMENTS, DATA, segmentsPathFixedPrefix) + .buildAsString(); Path segmentDataRepoPath = location.resolve(shardPath); String segmentDataLocalPath = String.format(Locale.ROOT, "%s/indices/%s/0/index", response.getShards()[0].getDataPath(), indexUuid); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 41350a0f553f4..26b30af4c2c50 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -324,13 +324,15 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception { final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class); final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(REMOTE_REPO_NAME); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); BlobPath shardLevelBlobPath = getShardLevelBlobPath( client(), remoteStoreEnabledIndexName, remoteStoreRepository.basePath(), "0", SEGMENTS, - LOCK_FILES + LOCK_FILES, + segmentsPathFixedPrefix ); BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath); String[] lockFiles; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index 774bffa10da4f..0bf40d0af01dc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -53,6 +53,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryCleanupResult; @@ -109,7 +110,8 @@ public TransportCleanupRepositoryAction( SnapshotsService snapshotsService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + RemoteStoreSettings remoteStoreSettings ) { super( CleanupRepositoryAction.NAME, @@ -122,7 +124,10 @@ public TransportCleanupRepositoryAction( ); this.repositoriesService = repositoriesService; this.snapshotsService = snapshotsService; - this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(() -> repositoriesService); + this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory( + () -> repositoriesService, + remoteStoreSettings.getSegmentsPathFixedPrefix() + ); // We add a state applier that will remove any dangling repository cleanup actions on cluster-manager failover. // This is safe to do since cleanups will increment the repository state id before executing any operations to prevent concurrent // operations from corrupting the repository. This is the same safety mechanism used by snapshot deletes. diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index e5f018aca5c0a..f418ceab38feb 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -766,6 +766,8 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_LOOKBACK_INTERVAL, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, + RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, + RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java index 899ff16c9d607..e686bb60140c0 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java @@ -20,6 +20,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import org.opensearch.index.remote.RemoteStorePathStrategy.ShardDataPathInput; +import org.opensearch.indices.RemoteStoreSettings; import java.io.IOException; import java.util.Collections; @@ -68,6 +69,7 @@ public class RemoteIndexPath implements ToXContentFragment { private final Iterable basePath; private final PathType pathType; private final PathHashAlgorithm pathHashAlgorithm; + private final RemoteStoreSettings remoteStoreSettings; /** * This keeps the map of paths that would be present in the content of the index path file. For eg - It is possible @@ -82,7 +84,8 @@ public RemoteIndexPath( Iterable basePath, PathType pathType, PathHashAlgorithm pathHashAlgorithm, - Map> pathCreationMap + Map> pathCreationMap, + RemoteStoreSettings remoteStoreSettings ) { if (Objects.isNull(pathCreationMap) || Objects.isNull(pathType) @@ -119,6 +122,7 @@ public RemoteIndexPath( this.pathType = pathType; this.pathHashAlgorithm = pathHashAlgorithm; this.pathCreationMap = pathCreationMap; + this.remoteStoreSettings = remoteStoreSettings; } @Override @@ -148,6 +152,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .shardId(Integer.toString(shardNo)) .dataCategory(dataCategory) .dataType(type) + .fixedPrefix( + dataCategory == TRANSLOG + ? remoteStoreSettings.getTranslogPathFixedPrefix() + : remoteStoreSettings.getSegmentsPathFixedPrefix() + ) .build(); builder.value(pathType.path(pathInput, pathHashAlgorithm).buildAsString()); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index d5617bdfd94a7..5878dff03acc2 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -25,6 +25,7 @@ import org.opensearch.gateway.remote.IndexMetadataUploadListener; import org.opensearch.gateway.remote.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreEnums.PathType; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -79,6 +80,7 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener { private final Settings settings; private final boolean isRemoteDataAttributePresent; private final boolean isTranslogSegmentRepoSame; + private final RemoteStoreSettings remoteStoreSettings; private final Supplier repositoriesService; private volatile TimeValue metadataUploadTimeout; @@ -89,7 +91,8 @@ public RemoteIndexPathUploader( ThreadPool threadPool, Settings settings, Supplier repositoriesService, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + RemoteStoreSettings remoteStoreSettings ) { super(threadPool, ThreadPool.Names.GENERIC); this.settings = Objects.requireNonNull(settings); @@ -100,6 +103,7 @@ public RemoteIndexPathUploader( Objects.requireNonNull(clusterSettings); metadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setMetadataUploadTimeout); + this.remoteStoreSettings = remoteStoreSettings; } @Override @@ -208,7 +212,8 @@ private void writePathToRemoteStore( basePath, pathType, hashAlgorithm, - pathCreationMap + pathCreationMap, + remoteStoreSettings ); String fileName = generateFileName(indexUUID, idxMD.getVersion(), remoteIndexPath.getVersion()); REMOTE_INDEX_PATH_FORMAT.writeAsyncWithUrgentPriority(remoteIndexPath, blobContainer, fileName, actionListener); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index 0af0d07e11597..c00e1a6a96abd 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -13,6 +13,7 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.hash.FNV1a; +import org.opensearch.core.common.Strings; import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import java.util.HashMap; @@ -107,7 +108,11 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return BlobPath.cleanPath().add(hashAlgorithm.hash(pathInput)).add(pathInput.basePath()).add(pathInput.fixedSubPath()); + String fixedPrefix = pathInput.fixedPrefix(); + return BlobPath.cleanPath() + .add(Strings.isNullOrEmpty(fixedPrefix) ? hashAlgorithm.hash(pathInput) : fixedPrefix + hashAlgorithm.hash(pathInput)) + .add(pathInput.basePath()) + .add(pathInput.fixedSubPath()); } @Override @@ -119,7 +124,10 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return pathInput.basePath().add(hashAlgorithm.hash(pathInput)).add(pathInput.fixedSubPath()); + String fixedPrefix = pathInput.fixedPrefix(); + return pathInput.basePath() + .add(Strings.isNullOrEmpty(fixedPrefix) ? hashAlgorithm.hash(pathInput) : fixedPrefix + hashAlgorithm.hash(pathInput)) + .add(pathInput.fixedSubPath()); } @Override diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index 7bead3e75f12e..ee28f47664614 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -83,10 +83,12 @@ public BlobPath generatePath(PathInput pathInput) { public static class PathInput { private final BlobPath basePath; private final String indexUUID; + private final String fixedPrefix; public PathInput(Builder builder) { this.basePath = Objects.requireNonNull(builder.basePath); this.indexUUID = Objects.requireNonNull(builder.indexUUID); + this.fixedPrefix = Objects.isNull(builder.fixedPrefix) ? "" : builder.fixedPrefix; } BlobPath basePath() { @@ -97,6 +99,10 @@ String indexUUID() { return indexUUID; } + String fixedPrefix() { + return fixedPrefix; + } + BlobPath fixedSubPath() { return BlobPath.cleanPath().add(indexUUID); } @@ -126,6 +132,7 @@ public void assertIsValid() { public static class Builder> { private BlobPath basePath; private String indexUUID; + private String fixedPrefix; public T basePath(BlobPath basePath) { this.basePath = basePath; @@ -137,6 +144,11 @@ public T indexUUID(String indexUUID) { return self(); } + public T fixedPrefix(String fixedPrefix) { + this.fixedPrefix = fixedPrefix; + return self(); + } + protected T self() { return (T) this; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 4b18df7a7829d..354ecd72b4b08 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2892,7 +2892,14 @@ public void restoreFromSnapshotAndRemoteStore( assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - storeRecovery.recoverFromSnapshotAndRemoteStore(this, repository, repositoriesService, listener, threadPool); + storeRecovery.recoverFromSnapshotAndRemoteStore( + this, + repository, + repositoriesService, + listener, + remoteStoreSettings.getSegmentsPathFixedPrefix(), + threadPool + ); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 8d689e8769728..8649da936d81c 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -367,6 +367,7 @@ void recoverFromSnapshotAndRemoteStore( Repository repository, RepositoriesService repositoriesService, ActionListener listener, + String segmentsPathFixedPrefix, ThreadPool threadPool ) { try { @@ -397,7 +398,8 @@ void recoverFromSnapshotAndRemoteStore( RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory( () -> repositoriesService, - threadPool + threadPool, + segmentsPathFixedPrefix ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( remoteStoreRepository, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 3f6f4eeeef87b..ea7e2506deec3 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -40,11 +40,17 @@ @PublicApi(since = "2.3.0") public class RemoteSegmentStoreDirectoryFactory implements IndexStorePlugin.DirectoryFactory { private final Supplier repositoriesService; + private final String segmentsPathFixedPrefix; private final ThreadPool threadPool; - public RemoteSegmentStoreDirectoryFactory(Supplier repositoriesService, ThreadPool threadPool) { + public RemoteSegmentStoreDirectoryFactory( + Supplier repositoriesService, + ThreadPool threadPool, + String segmentsPathFixedPrefix + ) { this.repositoriesService = repositoriesService; + this.segmentsPathFixedPrefix = segmentsPathFixedPrefix; this.threadPool = threadPool; } @@ -71,6 +77,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s .shardId(shardIdStr) .dataCategory(SEGMENTS) .dataType(DATA) + .fixedPrefix(segmentsPathFixedPrefix) .build(); // Derive the path for data directory of SEGMENTS BlobPath dataPath = pathStrategy.generatePath(dataPathInput); @@ -87,6 +94,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s .shardId(shardIdStr) .dataCategory(SEGMENTS) .dataType(METADATA) + .fixedPrefix(segmentsPathFixedPrefix) .build(); // Derive the path for metadata directory of SEGMENTS BlobPath mdPath = pathStrategy.generatePath(mdPathInput); @@ -98,7 +106,8 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s repositoryName, indexUUID, shardIdStr, - pathStrategy + pathStrategy, + segmentsPathFixedPrefix ); return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId); diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java index 993c1bbdf033f..40b540d30adc7 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java @@ -31,9 +31,11 @@ @PublicApi(since = "2.8.0") public class RemoteStoreLockManagerFactory { private final Supplier repositoriesService; + private final String segmentsPathFixedPrefix; - public RemoteStoreLockManagerFactory(Supplier repositoriesService) { + public RemoteStoreLockManagerFactory(Supplier repositoriesService, String segmentsPathFixedPrefix) { this.repositoriesService = repositoriesService; + this.segmentsPathFixedPrefix = segmentsPathFixedPrefix; } public RemoteStoreLockManager newLockManager( @@ -42,7 +44,7 @@ public RemoteStoreLockManager newLockManager( String shardId, RemoteStorePathStrategy pathStrategy ) { - return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy); + return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix); } public static RemoteStoreMetadataLockManager newLockManager( @@ -50,7 +52,8 @@ public static RemoteStoreMetadataLockManager newLockManager( String repositoryName, String indexUUID, String shardId, - RemoteStorePathStrategy pathStrategy + RemoteStorePathStrategy pathStrategy, + String segmentsPathFixedPrefix ) { try (Repository repository = repositoriesService.repository(repositoryName)) { assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; @@ -62,6 +65,7 @@ public static RemoteStoreMetadataLockManager newLockManager( .shardId(shardId) .dataCategory(SEGMENTS) .dataType(LOCK_FILES) + .fixedPrefix(segmentsPathFixedPrefix) .build(); BlobPath lockDirectoryPath = pathStrategy.generatePath(lockFilesPathInput); BlobContainer lockDirectoryBlobContainer = ((BlobStoreRepository) repository).blobStore().blobContainer(lockDirectoryPath); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index f6afc4c508720..66eea2a51b36a 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -392,6 +392,7 @@ public static TranslogTransferManager buildTranslogTransferManager( .shardId(shardIdStr) .dataCategory(TRANSLOG) .dataType(DATA) + .fixedPrefix(remoteStoreSettings.getTranslogPathFixedPrefix()) .build(); BlobPath dataPath = pathStrategy.generatePath(dataPathInput); RemoteStorePathStrategy.ShardDataPathInput mdPathInput = RemoteStorePathStrategy.ShardDataPathInput.builder() @@ -400,6 +401,7 @@ public static TranslogTransferManager buildTranslogTransferManager( .shardId(shardIdStr) .dataCategory(TRANSLOG) .dataType(METADATA) + .fixedPrefix(remoteStoreSettings.getTranslogPathFixedPrefix()) .build(); BlobPath mdPath = pathStrategy.generatePath(mdPathInput); BlobStoreTransferService transferService = new BlobStoreTransferService(blobStoreRepository.blobStore(), threadPool); diff --git a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java index 00d9ffdac6086..4bd0caa709436 100644 --- a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java +++ b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java @@ -164,6 +164,26 @@ public class RemoteStoreSettings { Setting.Property.NodeScope ); + /** + * Controls the fixed prefix for the translog path on remote store. + */ + public static final Setting CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX = Setting.simpleString( + "cluster.remote_store.translog.path.prefix", + "", + Property.NodeScope, + Property.Final + ); + + /** + * Controls the fixed prefix for the segments path on remote store. + */ + public static final Setting CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX = Setting.simpleString( + "cluster.remote_store.segments.path.prefix", + "", + Property.NodeScope, + Property.Final + ); + private volatile TimeValue clusterRemoteTranslogBufferInterval; private volatile int minRemoteSegmentMetadataFiles; private volatile TimeValue clusterRemoteTranslogTransferTimeout; @@ -175,6 +195,8 @@ public class RemoteStoreSettings { private static volatile boolean isPinnedTimestampsEnabled; private static volatile TimeValue pinnedTimestampsSchedulerInterval; private static volatile TimeValue pinnedTimestampsLookbackInterval; + private final String translogPathFixedPrefix; + private final String segmentsPathFixedPrefix; public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) { clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); @@ -216,6 +238,9 @@ public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) { pinnedTimestampsSchedulerInterval = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.get(settings); pinnedTimestampsLookbackInterval = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_LOOKBACK_INTERVAL.get(settings); isPinnedTimestampsEnabled = CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.get(settings); + + translogPathFixedPrefix = CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(settings); + segmentsPathFixedPrefix = CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(settings); } public TimeValue getClusterRemoteTranslogBufferInterval() { @@ -300,4 +325,12 @@ public static void setPinnedTimestampsLookbackInterval(TimeValue pinnedTimestamp public static boolean isPinnedTimestampsEnabled() { return isPinnedTimestampsEnabled; } + + public String getTranslogPathFixedPrefix() { + return translogPathFixedPrefix; + } + + public String getSegmentsPathFixedPrefix() { + return segmentsPathFixedPrefix; + } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 6ce7e2678d6d0..563d184ed26a6 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -791,6 +791,7 @@ protected Node( clusterService.getClusterSettings(), threadPool::relativeTimeInMillis ); + final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, settingsModule.getClusterSettings()); final RemoteClusterStateService remoteClusterStateService; final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; final RemoteIndexPathUploader remoteIndexPathUploader; @@ -799,7 +800,8 @@ protected Node( threadPool, settings, repositoriesServiceReference::get, - clusterService.getClusterSettings() + clusterService.getClusterSettings(), + remoteStoreSettings ); remoteClusterStateService = new RemoteClusterStateService( nodeEnvironment.nodeId(), @@ -871,12 +873,12 @@ protected Node( final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); - final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, settingsModule.getClusterSettings()); final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(settings, settingsModule.getClusterSettings()); final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get, - threadPool + threadPool, + remoteStoreSettings.getSegmentsPathFixedPrefix() ); final TaskResourceTrackingService taskResourceTrackingService = new TaskResourceTrackingService( @@ -1209,7 +1211,8 @@ protected Node( repositoryService, transportService, actionModule.getActionFilters(), - remoteStorePinnedTimestampService + remoteStorePinnedTimestampService, + remoteStoreSettings ); SnapshotShardsService snapshotShardsService = new SnapshotShardsService( settings, @@ -1419,6 +1422,7 @@ protected Node( b.bind(SnapshotsInfoService.class).toInstance(snapshotsInfoService); b.bind(GatewayMetaState.class).toInstance(gatewayMetaState); b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); + b.bind(RemoteStoreSettings.class).toInstance(remoteStoreSettings); { b.bind(PeerRecoverySourceService.class) .toInstance(new PeerRecoverySourceService(transportService, indicesService, recoverySettings)); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index d2df42fed6db5..321daaf72e28a 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -130,6 +130,7 @@ import org.opensearch.index.store.lockmanager.FileLockInfo; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.repositories.IndexId; @@ -420,6 +421,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final RecoverySettings recoverySettings; + private final RemoteStoreSettings remoteStoreSettings; + private final NamedXContentRegistry namedXContentRegistry; /** @@ -472,6 +475,7 @@ protected BlobStoreRepository( this.threadPool = clusterService.getClusterApplierService().threadPool(); this.clusterService = clusterService; this.recoverySettings = recoverySettings; + this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()); } @Override @@ -1296,7 +1300,8 @@ protected void releaseRemoteStoreLockAndCleanup( // related issue: https://github.com/opensearch-project/OpenSearch/issues/8469 RemoteSegmentStoreDirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( remoteStoreLockManagerFactory.getRepositoriesService(), - threadPool + threadPool, + remoteStoreSettings.getSegmentsPathFixedPrefix() ); remoteDirectoryCleanupAsync( remoteDirectoryFactory, diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 232dbe946d2ae..abaa85eb9eace 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -92,6 +92,7 @@ import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; @@ -226,12 +227,16 @@ public SnapshotsService( RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters, - @Nullable RemoteStorePinnedTimestampService remoteStorePinnedTimestampService + @Nullable RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + RemoteStoreSettings remoteStoreSettings ) { this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.repositoriesService = repositoriesService; - this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(() -> repositoriesService); + this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory( + () -> repositoriesService, + remoteStoreSettings.getSegmentsPathFixedPrefix() + ); this.threadPool = transportService.getThreadPool(); this.transportService = transportService; this.remoteStorePinnedTimestampService = remoteStorePinnedTimestampService; diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 5ac94281822b8..9972bbfff5d66 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -73,6 +73,7 @@ import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; import org.opensearch.index.remote.RemoteIndexPathUploader; +import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.fs.FsRepository; @@ -504,7 +505,15 @@ public void testDataOnlyNodePersistence() throws Exception { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + settings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); } else { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 835087632d0d0..5dec516480e59 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -62,6 +62,7 @@ import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; +import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; @@ -252,7 +253,15 @@ public void setup() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + settings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), namedWriteableRegistry ); } @@ -289,7 +298,15 @@ public void testFailInitializationWhenRemoteStateDisabled() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + settings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ) ); @@ -361,7 +378,15 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + settings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()) @@ -731,7 +756,15 @@ public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws I clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + settings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); @@ -2628,7 +2661,15 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + newSettings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof InternalRemoteRoutingTableService); @@ -2899,7 +2940,15 @@ private void initializeRoutingTable() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, newSettings, repositoriesServiceSupplier, clusterSettings)), + List.of( + new RemoteIndexPathUploader( + threadPool, + newSettings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), writableRegistry() ); } diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..8618d776bdf04 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -260,7 +260,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { writableRegistry(), () -> false, null, - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, ""), translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java index 8ddbd383756e7..2351d165b369f 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java @@ -9,12 +9,16 @@ package org.opensearch.index.remote; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; +import org.opensearch.indices.DefaultRemoteStoreSettings; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -40,7 +44,8 @@ public void testToXContentWithSegmentRepo() throws IOException { new BlobPath().add("djsd878ndjh").add("hcs87cj8"), PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_BASE64, - RemoteIndexPath.SEGMENT_PATH + RemoteIndexPath.SEGMENT_PATH, + DefaultRemoteStoreSettings.INSTANCE ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); xContentBuilder.startObject(); @@ -49,6 +54,28 @@ public void testToXContentWithSegmentRepo() throws IOException { String expected = "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}"; assertEquals(expected, xContentBuilder.toString()); + + // Fixed prefix + Settings settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".").build(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + indexPath = new RemoteIndexPath( + "djjsid73he8yd7usduh", + 2, + new BlobPath().add("djsd878ndjh").add("hcs87cj8"), + PathType.HASHED_PREFIX, + PathHashAlgorithm.FNV_1A_BASE64, + RemoteIndexPath.SEGMENT_PATH, + remoteStoreSettings + ); + xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); + xContentBuilder.startObject(); + xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + xContentBuilder.endObject(); + expected = + "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\".ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\".z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\".VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\".IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\".pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}"; + assertEquals(expected, xContentBuilder.toString()); + } /** @@ -61,7 +88,8 @@ public void testToXContentForTranslogRepoOnly() throws IOException { new BlobPath().add("djsd878ndjh").add("hcs87cj8"), PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_BASE64, - RemoteIndexPath.TRANSLOG_PATH + RemoteIndexPath.TRANSLOG_PATH, + DefaultRemoteStoreSettings.INSTANCE ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); xContentBuilder.startObject(); @@ -70,6 +98,27 @@ public void testToXContentForTranslogRepoOnly() throws IOException { String expected = "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}"; assertEquals(expected, xContentBuilder.toString()); + + // Fixed prefix + Settings settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".").build(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + indexPath = new RemoteIndexPath( + "djjsid73he8yd7usduh", + 2, + new BlobPath().add("djsd878ndjh").add("hcs87cj8"), + PathType.HASHED_PREFIX, + PathHashAlgorithm.FNV_1A_BASE64, + RemoteIndexPath.TRANSLOG_PATH, + remoteStoreSettings + ); + xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); + xContentBuilder.startObject(); + xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + xContentBuilder.endObject(); + expected = + "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\".2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\".dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\".PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\".NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}"; + assertEquals(expected, xContentBuilder.toString()); } /** @@ -85,7 +134,8 @@ public void testToXContentForBothRepos() throws IOException { new BlobPath().add("nxf9yv0").add("c3ejoi"), PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_BASE64, - pathCreationMap + pathCreationMap, + DefaultRemoteStoreSettings.INSTANCE ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); xContentBuilder.startObject(); @@ -94,9 +144,42 @@ public void testToXContentForBothRepos() throws IOException { String expected = "{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}"; assertEquals(expected, xContentBuilder.toString()); + + // Fixed prefix + Settings settings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), ".") + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), ".") + .build(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + indexPath = new RemoteIndexPath( + "csbdqiu8a7sdnjdks", + 3, + new BlobPath().add("nxf9yv0").add("c3ejoi"), + PathType.HASHED_PREFIX, + PathHashAlgorithm.FNV_1A_BASE64, + pathCreationMap, + remoteStoreSettings + ); + xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON); + xContentBuilder.startObject(); + xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + xContentBuilder.endObject(); + expected = + "{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\".kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\".p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\".gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\".Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\".HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\".igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\".uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\".TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\".QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\".sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\".d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\".zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"._s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\".tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}"; + assertEquals(expected, xContentBuilder.toString()); } - public void testRemoteIndexPathWithInvalidPathCreationMap() throws IOException { + public void testRemoteIndexPathWithInvalidPathCreationMap() { + Settings.Builder builder = Settings.builder(); + if (randomBoolean()) { + builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), "."); + } + if (randomBoolean()) { + builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), "."); + } + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(builder.build(), clusterSettings); IllegalArgumentException ex = assertThrows( IllegalArgumentException.class, () -> new RemoteIndexPath( @@ -105,7 +188,8 @@ public void testRemoteIndexPathWithInvalidPathCreationMap() throws IOException { new BlobPath().add("djsd878ndjh").add("hcs87cj8"), PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_BASE64, - new HashMap<>() + new HashMap<>(), + remoteStoreSettings ) ); assertEquals( @@ -124,6 +208,15 @@ public void testFromXContent() { } public void testInvalidPathCreationMap() { + Settings.Builder builder = Settings.builder(); + if (randomBoolean()) { + builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), "."); + } + if (randomBoolean()) { + builder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), "."); + } + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(builder.build(), clusterSettings); IllegalArgumentException ex = assertThrows( IllegalArgumentException.class, () -> new RemoteIndexPath( @@ -132,7 +225,8 @@ public void testInvalidPathCreationMap() { new BlobPath().add("djsd878ndjh").add("hcs87cj8"), PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_BASE64, - Map.of(TRANSLOG, List.of(LOCK_FILES)) + Map.of(TRANSLOG, List.of(LOCK_FILES)), + remoteStoreSettings ) ); assertEquals("pathCreationMap={TRANSLOG=[LOCK_FILES]} is having illegal combination of category and type", ex.getMessage()); diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java index e0a75f7296705..d6519d9db8ee6 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java @@ -25,6 +25,7 @@ import org.opensearch.gateway.remote.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; +import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -131,7 +132,8 @@ public void testInterceptWithNoRemoteDataAttributes() { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); List indexMetadataList = Mockito.mock(List.class); ActionListener actionListener = ActionListener.wrap( @@ -149,7 +151,8 @@ public void testInterceptWithEmptyIndexMetadataList() { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); ActionListener actionListener = ActionListener.wrap( @@ -166,7 +169,8 @@ public void testInterceptWithEmptyEligibleIndexMetadataList() { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); ActionListener actionListener = ActionListener.wrap( @@ -228,7 +232,8 @@ public void testInterceptWithSameRepo() throws IOException { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); ActionListener actionListener = ActionListener.wrap( @@ -251,7 +256,8 @@ public void testInterceptWithDifferentRepo() throws IOException { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); ActionListener actionListener = ActionListener.wrap( @@ -271,7 +277,8 @@ public void testInterceptWithLatchAwaitTimeout() throws IOException { threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); @@ -302,7 +309,8 @@ public void testInterceptWithInterruptedExceptionDuringLatchAwait() throws Excep threadPool, settings, () -> repositoriesService, - clusterSettings + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE ); remoteIndexPathUploader.start(); Settings settings = Settings.builder() diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java index abd115732c4db..331ebd0e3fcd4 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java @@ -220,4 +220,50 @@ public void testTranslogMetadataAllowedMinVersionOlder() { assertFalse(resolver.isTranslogMetadataEnabled()); } + public void testTranslogPathFixedPathSetting() { + + // Default settings + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + assertEquals("", remoteStoreSettings.getTranslogPathFixedPrefix()); + + // Any other random value + String randomPrefix = randomAlphaOfLengthBetween(2, 5); + settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), randomPrefix).build(); + remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + assertEquals(randomPrefix, remoteStoreSettings.getTranslogPathFixedPrefix()); + + // Set any other random value, the setting still points to the old value + clusterSettings.applySettings( + Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), randomAlphaOfLengthBetween(2, 5)) + .build() + ); + assertEquals(randomPrefix, remoteStoreSettings.getTranslogPathFixedPrefix()); + } + + public void testSegmentsPathFixedPathSetting() { + + // Default settings + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + assertEquals("", remoteStoreSettings.getSegmentsPathFixedPrefix()); + + // Any other random value + String randomPrefix = randomAlphaOfLengthBetween(2, 5); + settings = Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), randomPrefix).build(); + remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + assertEquals(randomPrefix, remoteStoreSettings.getSegmentsPathFixedPrefix()); + + // Set any other random value, the setting still points to the old value + clusterSettings.applySettings( + Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), randomAlphaOfLengthBetween(2, 5)) + .build() + ); + assertEquals(randomPrefix, remoteStoreSettings.getSegmentsPathFixedPrefix()); + + } } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java index cfcac5915837c..b2203d6eccd27 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java @@ -142,6 +142,7 @@ public void testGeneratePathForHashedPrefixType() { String shardId = String.valueOf(randomInt(100)); DataCategory dataCategory = TRANSLOG; DataType dataType = DATA; + String fixedPrefix = "."; String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId; // Translog Data @@ -151,11 +152,20 @@ public void testGeneratePathForHashedPrefixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -168,9 +178,10 @@ public void testGeneratePathForHashedPrefixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); + assertEquals(".DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); // Translog Metadata dataType = METADATA; @@ -180,11 +191,20 @@ public void testGeneratePathForHashedPrefixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -194,9 +214,10 @@ public void testGeneratePathForHashedPrefixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString()); + assertEquals(".oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString()); // Segment Data dataCategory = SEGMENTS; @@ -207,11 +228,20 @@ public void testGeneratePathForHashedPrefixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -221,9 +251,10 @@ public void testGeneratePathForHashedPrefixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); + assertEquals(".AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); // Segment Metadata dataType = METADATA; @@ -233,11 +264,20 @@ public void testGeneratePathForHashedPrefixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -247,9 +287,10 @@ public void testGeneratePathForHashedPrefixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString()); + assertEquals(".erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString()); // Segment Lockfiles dataType = LOCK_FILES; @@ -259,11 +300,20 @@ public void testGeneratePathForHashedPrefixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -273,12 +323,14 @@ public void testGeneratePathForHashedPrefixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString()); + assertEquals(".KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString()); } public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { + String fixedPrefix = "."; BlobPath blobPath = new BlobPath(); List pathList = getPathList(); for (String path : pathList) { @@ -298,12 +350,19 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -317,9 +376,10 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); - assertEquals("D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); + assertEquals(".D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); // Translog Metadata dataType = METADATA; @@ -329,12 +389,19 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -345,10 +412,11 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", + ".o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString() ); @@ -361,12 +429,19 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -377,9 +452,10 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); - assertEquals("A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); + assertEquals(".A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); // Segment Metadata dataType = METADATA; @@ -389,12 +465,19 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -405,10 +488,11 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", + ".e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString() ); @@ -420,12 +504,19 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + fixedPrefix + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -436,10 +527,11 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", + ".K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString() ); } @@ -455,6 +547,7 @@ public void testGeneratePathForHashedInfixType() { String shardId = String.valueOf(randomInt(100)); DataCategory dataCategory = TRANSLOG; DataType dataType = DATA; + String fixedPrefix = "."; String basePath = getPath(pathList); basePath = basePath.isEmpty() ? basePath : basePath.substring(0, basePath.length() - 1); @@ -465,9 +558,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); BlobPath result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - String expected = derivePath(basePath, pathInput); + String expected = derivePath(basePath, pathInput, fixedPrefix); String actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -481,9 +575,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/DgSI70IciXs/k2ijhe877d7yuhx7/10/translog/data/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/.DgSI70IciXs/k2ijhe877d7yuhx7/10/translog/data/"; actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -495,10 +590,11 @@ public void testGeneratePathForHashedInfixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = derivePath(basePath, pathInput); + expected = derivePath(basePath, pathInput, fixedPrefix); actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -509,9 +605,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/oKU5SjILiy4/k2ijhe877d7yuhx7/10/translog/metadata/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/.oKU5SjILiy4/k2ijhe877d7yuhx7/10/translog/metadata/"; actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -524,9 +621,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = derivePath(basePath, pathInput); + expected = derivePath(basePath, pathInput, fixedPrefix); actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -537,9 +635,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/AUBRfCIuWdk/k2ijhe877d7yuhx7/10/segments/data/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/.AUBRfCIuWdk/k2ijhe877d7yuhx7/10/segments/data/"; actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -551,9 +650,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = derivePath(basePath, pathInput); + expected = derivePath(basePath, pathInput, fixedPrefix); actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -564,9 +664,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/erwR-G735Uw/k2ijhe877d7yuhx7/10/segments/metadata/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/.erwR-G735Uw/k2ijhe877d7yuhx7/10/segments/metadata/"; actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -578,9 +679,10 @@ public void testGeneratePathForHashedInfixType() { .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = derivePath(basePath, pathInput); + expected = derivePath(basePath, pathInput, fixedPrefix); actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); @@ -591,14 +693,16 @@ public void testGeneratePathForHashedInfixType() { .shardId(fixedShardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/KeYDIk0mJXI/k2ijhe877d7yuhx7/10/segments/lock_files/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/.KeYDIk0mJXI/k2ijhe877d7yuhx7/10/segments/lock_files/"; actual = result.buildAsString(); assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected)); } public void testGeneratePathForSnapshotShardPathInput() { + String fixedPrefix = "snap"; BlobPath blobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij"); String indexUUID = "dsdkjsu8832njn"; String shardId = "10"; @@ -606,6 +710,7 @@ public void testGeneratePathForSnapshotShardPathInput() { .basePath(blobPath) .indexUUID(indexUUID) .shardId(shardId) + .fixedPrefix(fixedPrefix) .build(); // FIXED PATH @@ -616,34 +721,34 @@ public void testGeneratePathForSnapshotShardPathInput() { // HASHED_PREFIX - FNV_1A_COMPOSITE_1 result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); - expected = "_11001000010110/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/"; + expected = "snap_11001000010110/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/"; actual = result.buildAsString(); assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected); // HASHED_PREFIX - FNV_1A_BASE64 result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - expected = "_yFiSl_VGGM/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/"; + expected = "snap_yFiSl_VGGM/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/indices/dsdkjsu8832njn/10/"; actual = result.buildAsString(); assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected); // HASHED_INFIX - FNV_1A_COMPOSITE_1 result = HASHED_INFIX.path(pathInput, FNV_1A_COMPOSITE_1); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_11001000010110/indices/dsdkjsu8832njn/10/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/snap_11001000010110/indices/dsdkjsu8832njn/10/"; actual = result.buildAsString(); assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected); // HASHED_INFIX - FNV_1A_BASE64 result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64); - expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/_yFiSl_VGGM/indices/dsdkjsu8832njn/10/"; + expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/snap_yFiSl_VGGM/indices/dsdkjsu8832njn/10/"; actual = result.buildAsString(); assertEquals(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual, expected); } - private String derivePath(String basePath, ShardDataPathInput pathInput) { + private String derivePath(String basePath, ShardDataPathInput pathInput, String fixedPrefix) { return "".equals(basePath) ? String.join( SEPARATOR, - FNV_1A_BASE64.hash(pathInput), + fixedPrefix + FNV_1A_BASE64.hash(pathInput), pathInput.indexUUID(), pathInput.shardId(), pathInput.dataCategory().getName(), @@ -652,7 +757,7 @@ private String derivePath(String basePath, ShardDataPathInput pathInput) { : String.join( SEPARATOR, basePath, - FNV_1A_BASE64.hash(pathInput), + fixedPrefix + FNV_1A_BASE64.hash(pathInput), pathInput.indexUUID(), pathInput.shardId(), pathInput.dataCategory().getName(), diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index cad5e47531cc6..95b5d4456baf6 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -57,7 +57,7 @@ public void setup() { repositoriesService = mock(RepositoriesService.class); threadPool = mock(ThreadPool.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier, threadPool); + remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(repositoriesServiceSupplier, threadPool, ""); } public void testNewDirectory() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java index de3dfbdaa4778..37ae0484d5ee1 100644 --- a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java @@ -42,7 +42,7 @@ public void setup() throws IOException { repositoriesServiceSupplier = mock(Supplier.class); repositoriesService = mock(RepositoriesService.class); when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); - remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(repositoriesServiceSupplier); + remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(repositoriesServiceSupplier, ""); } public void testNewLockManager() throws IOException { diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index cb0a36c870d07..97d4e576d692b 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -62,6 +62,7 @@ import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleListener; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; @@ -148,6 +149,10 @@ private RepositoriesService createRepositoriesServiceWithMockedClusterService(Cl when(currentClusterState.getNodes()).thenReturn(nodes); when(clusterService.state()).thenReturn(currentClusterState); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + RepositoriesService repositoriesService = new RepositoriesService( Settings.EMPTY, clusterService, diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java index 958a499ada167..970e375055bc7 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -23,6 +23,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.store.RemoteBufferedOutputDirectory; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; @@ -59,13 +60,16 @@ protected Collection> getPlugins() { protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepository) throws IOException { final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class); final BlobStoreRepository remoteStorerepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepository); + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(clusterService.getSettings()); BlobPath shardLevelBlobPath = getShardLevelBlobPath( client(), remoteStoreIndex, remoteStorerepository.basePath(), "0", SEGMENTS, - LOCK_FILES + LOCK_FILES, + segmentsPathFixedPrefix ); BlobContainer blobContainer = remoteStorerepository.blobStore().blobContainer(shardLevelBlobPath); try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index fd84c6b48df8a..71a2b42c9847a 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2013,7 +2013,8 @@ public void onFailure(final Exception e) { repositoriesService, transportService, actionFilters, - null + null, + DefaultRemoteStoreSettings.INSTANCE ); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); @@ -2070,7 +2071,7 @@ public void onFailure(final Exception e) { emptyMap(), null, emptyMap(), - new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool), + new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool, ""), repositoriesServiceReference::get, null, new RemoteStoreStatsTrackerFactory(clusterService, settings), @@ -2366,7 +2367,8 @@ public void onFailure(final Exception e) { snapshotsService, threadPool, actionFilters, - indexNameExpressionResolver + indexNameExpressionResolver, + DefaultRemoteStoreSettings.INSTANCE ) ); actions.put( diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java index e290cdee836b1..027b1bef84e7f 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/BlobStoreTestUtil.java @@ -51,6 +51,8 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; @@ -463,6 +465,9 @@ private static ClusterService mockClusterService(ClusterState initialState) { return null; }).when(clusterService).addStateApplier(any(ClusterStateApplier.class)); when(clusterApplierService.threadPool()).thenReturn(threadPool); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); return clusterService; } diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 196aaf23df3c6..a8bb10fe20752 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -66,6 +66,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexModule; import org.opensearch.index.store.RemoteBufferedOutputDirectory; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.NodeClosedException; import org.opensearch.plugins.Plugin; @@ -553,13 +554,15 @@ protected void assertDocCount(String index, long count) { protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepositoryName) throws IOException { final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class); final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepositoryName); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); BlobPath shardLevelBlobPath = getShardLevelBlobPath( client(), remoteStoreIndex, remoteStoreRepository.basePath(), "0", SEGMENTS, - LOCK_FILES + LOCK_FILES, + segmentsPathFixedPrefix ); BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath); try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) { diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 2bf610974767b..70f7812f1db19 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -399,6 +399,10 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static Boolean prefixModeVerificationEnable; + private static Boolean translogPathFixedPrefix; + + private static Boolean segmentsPathFixedPrefix; + private Path remoteStoreRepositoryPath; private ReplicationType randomReplicationType; @@ -408,6 +412,8 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { @BeforeClass public static void beforeClass() throws Exception { prefixModeVerificationEnable = randomBoolean(); + translogPathFixedPrefix = randomBoolean(); + segmentsPathFixedPrefix = randomBoolean(); testClusterRule.beforeClass(); } @@ -2585,6 +2591,12 @@ protected void updateRepository(String repoName, String type, Settings.Builder s putRepository(clusterAdmin(), repoName, type, null, settings); } + public Settings getNodeSettings() { + InternalTestCluster internalTestCluster = internalCluster(); + ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getClusterManagerName()); + return clusterService.getSettings(); + } + public static void putRepository(ClusterAdminClient adminClient, String repoName, String type, Settings.Builder settings) { assertAcked(putRepositoryRequestBuilder(adminClient, repoName, type, true, settings, null, false)); } @@ -2887,6 +2899,8 @@ private static Settings buildRemoteStoreNodeAttributes( settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values())); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean()); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false); + settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : ""); + settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : ""); return settings.build(); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index e07d7ef3c5a7b..b180187303a60 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1817,7 +1817,8 @@ public static BlobPath getShardLevelBlobPath( BlobPath basePath, String shardId, RemoteStoreEnums.DataCategory dataCategory, - RemoteStoreEnums.DataType dataType + RemoteStoreEnums.DataType dataType, + String fixedPrefix ) { String indexUUID = client.admin() .indices() @@ -1842,6 +1843,7 @@ public static BlobPath getShardLevelBlobPath( .shardId(shardId) .dataCategory(dataCategory) .dataType(dataType) + .fixedPrefix(fixedPrefix) .build(); return type.path(pathInput, hashAlgorithm); } From 70ba5a11ef16b08284e5c7477ba87058be2b4e9f Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Tue, 3 Sep 2024 16:47:24 +0530 Subject: [PATCH 020/357] [Snapshot V2] Fix for create snapshot build and test failures (#15602) --------- Signed-off-by: Anshu Agarwal Signed-off-by: Sachin Kale Co-authored-by: Anshu Agarwal Co-authored-by: Sachin Kale --- .../repositories/FilterRepository.java | 21 ++++++++++++ .../opensearch/repositories/Repository.java | 24 ++++++++++++++ .../blobstore/BlobStoreRepository.java | 22 +++++++++++++ .../opensearch/snapshots/SnapshotInfo.java | 32 +++++++++++++++++++ .../snapshots/SnapshotsService.java | 3 +- .../RepositoriesServiceTests.java | 13 ++++++++ .../index/shard/RestoreOnlyRepository.java | 13 ++++++++ .../test/OpenSearchIntegTestCase.java | 2 +- 8 files changed, 127 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 114cd0260fcca..188d61e00f206 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -97,6 +97,27 @@ public void getRepositoryData(ActionListener listener) { in.getRepositoryData(listener); } + @Override + public void finalizeSnapshot( + ShardGenerations shardGenerations, + long repositoryStateId, + Metadata clusterMetadata, + SnapshotInfo snapshotInfo, + Version repositoryMetaVersion, + Function stateTransformer, + ActionListener listener + ) { + in.finalizeSnapshot( + shardGenerations, + repositoryStateId, + clusterMetadata, + snapshotInfo, + repositoryMetaVersion, + stateTransformer, + listener + ); + } + @Override public void finalizeSnapshot( ShardGenerations shardGenerations, diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 637503d3f54df..136e9fb26c3dc 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -139,6 +139,30 @@ default Repository create(RepositoryMetadata metadata, Function listener); + /** + * Finalizes snapshotting process + *

+ * This method is called on cluster-manager after all shards are snapshotted. + * + * @param shardGenerations updated shard generations + * @param repositoryStateId the unique id identifying the state of the repository when the snapshot began + * @param clusterMetadata cluster metadata + * @param snapshotInfo SnapshotInfo instance to write for this snapshot + * @param repositoryMetaVersion version of the updated repository metadata to write + * @param stateTransformer a function that filters the last cluster state update that the snapshot finalization will execute and + * is used to remove any state tracked for the in-progress snapshot from the cluster state + * @param listener listener to be invoked with the new {@link RepositoryData} after completing the snapshot + */ + void finalizeSnapshot( + ShardGenerations shardGenerations, + long repositoryStateId, + Metadata clusterMetadata, + SnapshotInfo snapshotInfo, + Version repositoryMetaVersion, + Function stateTransformer, + ActionListener listener + ); + /** * Finalizes snapshotting process *

diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 321daaf72e28a..0a877a9ccea4b 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2024,6 +2024,28 @@ private DeleteResult cleanUpStaleSnapshotShardPathsFile(List matchingSha return new DeleteResult(matchingShardPaths.size(), totalBytes); } + @Override + public void finalizeSnapshot( + final ShardGenerations shardGenerations, + final long repositoryStateId, + final Metadata clusterMetadata, + SnapshotInfo snapshotInfo, + Version repositoryMetaVersion, + Function stateTransformer, + final ActionListener listener + ) { + finalizeSnapshot( + shardGenerations, + repositoryStateId, + clusterMetadata, + snapshotInfo, + repositoryMetaVersion, + stateTransformer, + Priority.NORMAL, + listener + ); + } + @Override public void finalizeSnapshot( final ShardGenerations shardGenerations, diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java index 7558c4456109e..bc33928acf3da 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java @@ -354,6 +354,38 @@ public SnapshotInfo(SnapshotsInProgress.Entry entry) { ); } + public SnapshotInfo( + SnapshotId snapshotId, + List indices, + List dataStreams, + long startTime, + String reason, + long endTime, + int totalShards, + List shardFailures, + Boolean includeGlobalState, + Map userMetadata, + Boolean remoteStoreIndexShallowCopy + ) { + this( + snapshotId, + indices, + dataStreams, + snapshotState(reason, shardFailures), + reason, + Version.CURRENT, + startTime, + endTime, + totalShards, + totalShards - shardFailures.size(), + shardFailures, + includeGlobalState, + userMetadata, + remoteStoreIndexShallowCopy, + 0 + ); + } + public SnapshotInfo( SnapshotId snapshotId, List indices, diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index abaa85eb9eace..8b595fb7135c1 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -205,6 +205,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus Setting.Property.Dynamic ); + private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__"; /** * Setting to specify the maximum number of shards that can be included in the result for the snapshot status * API call. Note that it does not apply to V2-shallow snapshots. @@ -216,8 +217,6 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus Setting.Property.NodeScope, Setting.Property.Dynamic ); - - private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = ":"; private volatile int maxConcurrentOperations; public SnapshotsService( diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 97d4e576d692b..4cd822c7d583b 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -681,6 +681,19 @@ public void finalizeSnapshot( listener.onResponse(null); } + @Override + public void finalizeSnapshot( + ShardGenerations shardGenerations, + long repositoryStateId, + Metadata clusterMetadata, + SnapshotInfo snapshotInfo, + Version repositoryMetaVersion, + Function stateTransformer, + ActionListener listener + ) { + listener.onResponse(null); + } + @Override public void deleteSnapshots( Collection snapshotIds, diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index 1ca1a6969ab2d..846f54bd13b40 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -118,6 +118,19 @@ public void getRepositoryData(ActionListener listener) { ); } + @Override + public void finalizeSnapshot( + ShardGenerations shardGenerations, + long repositoryStateId, + Metadata clusterMetadata, + SnapshotInfo snapshotInfo, + Version repositoryMetaVersion, + Function stateTransformer, + ActionListener listener + ) { + listener.onResponse(null); + } + @Override public void finalizeSnapshot( ShardGenerations shardGenerations, diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 70f7812f1db19..7d1fe1b000d20 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2898,7 +2898,7 @@ private static Settings buildRemoteStoreNodeAttributes( } settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values())); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean()); - settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false); + settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : ""); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : ""); return settings.build(); From 0f53bf9762f758505a814543ac93c5aaa7bf8e74 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:29:56 +0530 Subject: [PATCH 021/357] Change version to V_2_17_0 (#15596) Signed-off-by: Lakshya Taragi --- .../main/java/org/opensearch/OpenSearchServerException.java | 2 +- .../cluster/snapshots/status/SnapshotsStatusRequest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index cc3ceeb7942d3..a1fc61834f69b 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -1215,7 +1215,7 @@ public static void registerExceptions() { org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class, org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new, 175, - V_3_0_0 + V_2_17_0 ) ); registerExceptionHandle( diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index 87e863f8dd286..6c3438fbe58f6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -78,7 +78,7 @@ public SnapshotsStatusRequest(StreamInput in) throws IOException { repository = in.readString(); snapshots = in.readStringArray(); ignoreUnavailable = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { indices = in.readOptionalStringArray(); } } @@ -89,7 +89,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(repository); out.writeStringArray(snapshots); out.writeBoolean(ignoreUnavailable); - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalStringArray(indices); } } From 82762d4b1a8700eb05708d223ba6b50c0c2fbc3c Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 3 Sep 2024 17:30:10 +0530 Subject: [PATCH 022/357] Add cluster state checksum in manifest (#15218) * Add cluster state checksum in manifest for remote state and routing table publication Signed-off-by: Himshikha Gupta Co-authored-by: Bukhtawar Khan --- CHANGELOG.md | 1 + .../stream/BufferedChecksumStreamOutput.java | 80 +++ .../core/common/io/stream/StreamOutput.java | 2 +- .../common/io/stream/VerifiableWriteable.java | 20 + .../remote/RemoteRoutingTableServiceIT.java | 1 + .../remote/RemoteStatePublicationIT.java | 1 + .../opensearch/cluster/AbstractDiffable.java | 5 + .../org/opensearch/cluster/ClusterState.java | 30 +- .../org/opensearch/cluster/DiffableUtils.java | 12 + .../cluster/block/ClusterBlock.java | 8 +- .../cluster/block/ClusterBlocks.java | 9 +- .../coordination/CoordinationMetadata.java | 15 +- .../cluster/metadata/IndexMetadata.java | 31 +- .../metadata/IndexTemplateMetadata.java | 15 +- .../opensearch/cluster/metadata/Metadata.java | 2 +- .../cluster/metadata/TemplatesMetadata.java | 14 +- .../cluster/node/DiscoveryNode.java | 31 +- .../cluster/node/DiscoveryNodes.java | 55 +- .../cluster/routing/IndexRoutingTable.java | 17 +- .../routing/IndexShardRoutingTable.java | 23 + .../cluster/routing/RoutingTable.java | 15 +- .../common/settings/ClusterSettings.java | 1 + .../remote/ClusterMetadataManifest.java | 62 ++- .../gateway/remote/ClusterStateChecksum.java | 485 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 190 +++++++ .../gateway/remote/RemoteManifestManager.java | 6 +- .../model/RemoteClusterMetadataManifest.java | 1 + .../cluster/block/ClusterBlocksTests.java | 55 ++ .../CoordinationMetadataTests.java | 31 ++ .../cluster/metadata/IndexMetadataTests.java | 81 +++ .../metadata/IndexTemplateMetadataTests.java | 47 ++ .../cluster/node/DiscoveryNodeTests.java | 39 ++ .../cluster/node/DiscoveryNodesTests.java | 26 + .../routing/IndexShardRoutingTableTests.java | 22 + .../cluster/routing/RoutingTableTests.java | 1 + .../remote/ClusterMetadataManifestTests.java | 123 +++-- .../remote/ClusterStateChecksumTests.java | 222 ++++++++ .../RemoteClusterStateServiceTests.java | 367 ++++++++++++- 38 files changed, 2085 insertions(+), 61 deletions(-) create mode 100644 libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java create mode 100644 server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index eeb5f952b185f..93a98de2c5af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) +- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java index 422f956c0cd47..18bd53dc5d77c 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/BufferedChecksumStreamOutput.java @@ -33,9 +33,18 @@ package org.opensearch.core.common.io.stream; import org.apache.lucene.store.BufferedChecksum; +import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; import java.util.zip.CRC32; import java.util.zip.Checksum; @@ -90,4 +99,75 @@ public void reset() throws IOException { public void resetDigest() { digest.reset(); } + + @Override + public void writeMap(@Nullable Map map) throws IOException { + Map newMap = new TreeMap<>(map); + writeGenericValue(newMap); + } + + @Override + public void writeMap(Map map, final Writeable.Writer keyWriter, final Writeable.Writer valueWriter) + throws IOException { + writeVInt(map.size()); + map.keySet().stream().sorted().forEachOrdered(key -> { + try { + keyWriter.write(this, key); + valueWriter.write(this, map.get(key)); + } catch (IOException e) { + throw new RuntimeException("Failed to write map values.", e); + } + }); + } + + public void writeMapValues(Map map, final Writeable.Writer valueWriter) throws IOException { + writeVInt(map.size()); + map.keySet().stream().sorted().forEachOrdered(key -> { + try { + valueWriter.write(this, map.get(key)); + } catch (IOException e) { + throw new RuntimeException("Failed to write map values.", e); + } + }); + } + + @Override + public void writeStringArray(String[] array) throws IOException { + String[] copyArray = Arrays.copyOf(array, array.length); + Arrays.sort(copyArray); + super.writeStringArray(copyArray); + } + + @Override + public void writeVLongArray(long[] values) throws IOException { + long[] copyValues = Arrays.copyOf(values, values.length); + Arrays.sort(copyValues); + super.writeVLongArray(copyValues); + } + + @Override + public void writeCollection(final Collection collection) throws IOException { + List sortedList = collection.stream().sorted().collect(Collectors.toList()); + super.writeCollection(sortedList, (o, v) -> v.writeTo(o)); + } + + @Override + public void writeStringCollection(final Collection collection) throws IOException { + List listCollection = new ArrayList<>(collection); + Collections.sort(listCollection); + writeCollection(listCollection, StreamOutput::writeString); + } + + @Override + public void writeOptionalStringCollection(final Collection collection) throws IOException { + if (collection != null) { + List listCollection = new ArrayList<>(collection); + Collections.sort(listCollection); + writeBoolean(true); + writeCollection(listCollection, StreamOutput::writeString); + } else { + writeBoolean(false); + } + } + } diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index b7599265aece3..867fc99cc6a56 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -633,7 +633,7 @@ public final void writeMapOfLists(final Map> map, final Writer * @param keyWriter The key writer * @param valueWriter The value writer */ - public final void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { + public void writeMap(final Map map, final Writer keyWriter, final Writer valueWriter) throws IOException { writeVInt(map.size()); for (final Map.Entry entry : map.entrySet()) { keyWriter.write(this, entry.getKey()); diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java new file mode 100644 index 0000000000000..6683606e27195 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/VerifiableWriteable.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.common.io.stream; + +import java.io.IOException; + +/** + * Provides a method for serialization which will give ordered stream, creating same byte array on every invocation. + * This should be invoked with a stream that provides ordered serialization. + */ +public interface VerifiableWriteable extends Writeable { + + void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException; +} diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java index 9b80e4f907070..3c0c9a0611439 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java @@ -67,6 +67,7 @@ protected Settings nodeSettings(int nodeOrdinal) { ) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) .put(REMOTE_PUBLICATION_EXPERIMENTAL, true) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index f237a81942e1a..3a3e023db6446 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -90,6 +90,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) .build(); } diff --git a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java index 74af3472433ba..770a0c171e084 100644 --- a/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java +++ b/server/src/main/java/org/opensearch/cluster/AbstractDiffable.java @@ -83,6 +83,11 @@ private static class CompleteDiff> implements Diff { this.part = part; } + @Override + public String toString() { + return "CompleteDiff{" + "part=" + part + '}'; + } + /** * Creates simple diff without changes */ diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 9e63f961d241d..054233305726d 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -156,7 +156,7 @@ default boolean isPrivate() { } - private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); + public static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); public static final String UNKNOWN_UUID = "_na_"; @@ -839,6 +839,34 @@ private static class ClusterStateDiff implements Diff { minimumClusterManagerNodesOnPublishingClusterManager = after.minimumClusterManagerNodesOnPublishingClusterManager; } + @Override + public String toString() { + return new StringBuilder().append("ClusterStateDiff{toVersion=") + .append(toVersion) + .append(", fromUuid='") + .append(fromUuid) + .append('\'') + .append(", toUuid='") + .append(toUuid) + .append('\'') + .append(", clusterName=") + .append(clusterName) + .append(", routingTable=") + .append(routingTable) + .append(", nodes=") + .append(nodes) + .append(", metadata=") + .append(metadata) + .append(", blocks=") + .append(blocks) + .append(", customs=") + .append(customs) + .append(", minimumClusterManagerNodesOnPublishingClusterManager=") + .append(minimumClusterManagerNodesOnPublishingClusterManager) + .append("}") + .toString(); + } + ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { clusterName = new ClusterName(in); fromUuid = in.readString(); diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index d21cd354bf659..4b5dcaa52cc50 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -271,6 +271,18 @@ public Map getUpserts() { return upserts; } + @Override + public String toString() { + return new StringBuilder().append("MapDiff{deletes=") + .append(deletes) + .append(", diffs=") + .append(diffs) + .append(", upserts=") + .append(upserts) + .append("}") + .toString(); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(deletes, (o, v) -> keySerializer.writeKey(v, o)); diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index 5fa897c0b1185..7c0a7a2a6b837 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -52,7 +52,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class ClusterBlock implements Writeable, ToXContentFragment { +public class ClusterBlock implements Writeable, ToXContentFragment, Comparable { private final int id; @Nullable @@ -217,7 +217,13 @@ public int hashCode() { return Objects.hash(id, uuid); } + @Override + public int compareTo(ClusterBlock block) { + return Integer.compare(block.id(), this.id()); + } + public boolean isAllowReleaseResources() { return allowReleaseResources; } + } diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 02a20b7681ba7..615ea18315cd1 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -39,8 +39,10 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.set.Sets; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.rest.RestStatus; import java.io.IOException; @@ -62,7 +64,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class ClusterBlocks extends AbstractDiffable { +public class ClusterBlocks extends AbstractDiffable implements VerifiableWriteable { public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), Map.of()); private final Set global; @@ -303,6 +305,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indicesBlocks, StreamOutput::writeString, (o, s) -> writeBlockSet(s, o)); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + writeTo(out); + } + private static void writeBlockSet(Set blocks, StreamOutput out) throws IOException { out.writeCollection(blocks); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java index 53398d6f3f98f..d7291a3689192 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationMetadata.java @@ -35,8 +35,10 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.set.Sets; import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; @@ -59,7 +61,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class CoordinationMetadata implements Writeable, ToXContentFragment { +public class CoordinationMetadata implements VerifiableWriteable, ToXContentFragment { public static final CoordinationMetadata EMPTY_METADATA = builder().build(); @@ -149,6 +151,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(votingConfigExclusions); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + writeTo(out); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.field(TERM_PARSE_FIELD.getPreferredName(), term) @@ -272,7 +279,7 @@ public CoordinationMetadata build() { * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class VotingConfigExclusion implements Writeable, ToXContentFragment { + public static class VotingConfigExclusion implements Writeable, ToXContentFragment, Comparable { public static final String MISSING_VALUE_MARKER = "_absent_"; private final String nodeId; private final String nodeName; @@ -361,6 +368,10 @@ public String toString() { return sb.toString(); } + @Override + public int compareTo(VotingConfigExclusion votingConfigExclusion) { + return votingConfigExclusion.getNodeId().compareTo(this.getNodeId()); + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 405e9b3aabdb6..bb470ea9e4ab8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -54,8 +54,10 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.Assertions; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -88,6 +90,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.function.Function; import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; @@ -103,7 +106,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexMetadata implements Diffable, ToXContentFragment { +public class IndexMetadata implements Diffable, ToXContentFragment, VerifiableWriteable { public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock( 5, @@ -1264,6 +1267,32 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeString(index.getName()); // uuid will come as part of settings + out.writeLong(version); + out.writeVLong(mappingVersion); + out.writeVLong(settingsVersion); + out.writeVLong(aliasesVersion); + out.writeInt(routingNumShards); + out.writeByte(state.id()); + writeSettingsToStream(settings, out); + out.writeVLongArray(primaryTerms); + out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMap( + inSyncAllocationIds, + StreamOutput::writeVInt, + (stream, val) -> DiffableUtils.StringSetValueSerializer.getInstance().write(new TreeSet<>(val), stream) + ); + out.writeMapValues(rolloverInfos, (stream, val) -> val.writeTo(stream)); + out.writeBoolean(isSystem); + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { + out.writeOptionalWriteable(context); + } + } + public boolean isSystem() { return isSystem; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java index 3d532208bcfe2..b09acc54653c4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexTemplateMetadata.java @@ -44,8 +44,10 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -67,7 +69,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexTemplateMetadata extends AbstractDiffable { +public class IndexTemplateMetadata extends AbstractDiffable implements VerifiableWriteable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexTemplateMetadata.class); @@ -257,6 +259,17 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(version); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeString(name); + out.writeInt(order); + out.writeStringCollection(patterns); + Settings.writeSettingsToStream(settings, out); + out.writeMap(mappings, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); + out.writeOptionalVInt(version); + } + @Override public String toString() { try { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 5ace5b7bc2460..600f408cc963b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -253,7 +253,7 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio public static final String GLOBAL_STATE_FILE_PREFIX = "global-"; - private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); + public static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); private final String clusterUUID; private final boolean clusterUUIDCommitted; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index d31a51628eac4..c01cd8f373c72 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -10,7 +10,9 @@ import org.opensearch.cluster.AbstractDiffable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -27,7 +29,7 @@ * @opensearch.api */ @PublicApi(since = "2.15.0") -public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { +public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment, VerifiableWriteable { public static TemplatesMetadata EMPTY_METADATA = builder().build(); private final Map templates; @@ -65,6 +67,11 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeMapValues(templates, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -80,6 +87,11 @@ public int hashCode() { return templates != null ? templates.hashCode() : 0; } + @Override + public String toString() { + return "TemplatesMetadata{" + "templates=" + templates + '}'; + } + /** * Builder for the templates metadata * diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 653f81830ed17..50086d458fab2 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -37,9 +37,10 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -71,7 +72,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiscoveryNode implements Writeable, ToXContentFragment { +public class DiscoveryNode implements VerifiableWriteable, ToXContentFragment { static final String COORDINATING_ONLY = "coordinating_only"; @@ -358,17 +359,33 @@ public DiscoveryNode(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { + writeNodeDetails(out); + + out.writeVInt(attributes.size()); + for (Map.Entry entry : attributes.entrySet()) { + out.writeString(entry.getKey()); + out.writeString(entry.getValue()); + } + writeRolesAndVersion(out); + } + + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + writeNodeDetails(out); + out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString); + writeRolesAndVersion(out); + } + + private void writeNodeDetails(StreamOutput out) throws IOException { out.writeString(nodeName); out.writeString(nodeId); out.writeString(ephemeralId); out.writeString(hostName); out.writeString(hostAddress); address.writeTo(out); - out.writeVInt(attributes.size()); - for (Map.Entry entry : attributes.entrySet()) { - out.writeString(entry.getKey()); - out.writeString(entry.getValue()); - } + } + + private void writeRolesAndVersion(StreamOutput out) throws IOException { out.writeVInt(roles.size()); for (final DiscoveryNodeRole role : roles) { final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion()); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index 2ebcd8096893d..ba81012e386d7 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -41,8 +41,10 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.common.transport.TransportAddress; import java.io.IOException; @@ -66,7 +68,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiscoveryNodes extends AbstractDiffable implements Iterable { +public class DiscoveryNodes extends AbstractDiffable implements Iterable, VerifiableWriteable { public static final DiscoveryNodes EMPTY_NODES = builder().build(); @@ -688,16 +690,59 @@ public String shortSummary() { @Override public void writeTo(StreamOutput out) throws IOException { + writeClusterManager(out); + out.writeVInt(nodes.size()); + for (DiscoveryNode node : this) { + node.writeTo(out); + } + } + + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + writeClusterManager(out); + out.writeMapValues(nodes, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + } + + private void writeClusterManager(StreamOutput out) throws IOException { if (clusterManagerNodeId == null) { out.writeBoolean(false); } else { out.writeBoolean(true); out.writeString(clusterManagerNodeId); } - out.writeVInt(nodes.size()); - for (DiscoveryNode node : this) { - node.writeTo(out); - } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DiscoveryNodes that = (DiscoveryNodes) o; + return Objects.equals(nodes, that.nodes) + && Objects.equals(dataNodes, that.dataNodes) + && Objects.equals(clusterManagerNodes, that.clusterManagerNodes) + && Objects.equals(ingestNodes, that.ingestNodes) + && Objects.equals(clusterManagerNodeId, that.clusterManagerNodeId) + && Objects.equals(localNodeId, that.localNodeId) + && Objects.equals(minNonClientNodeVersion, that.minNonClientNodeVersion) + && Objects.equals(maxNonClientNodeVersion, that.maxNonClientNodeVersion) + && Objects.equals(maxNodeVersion, that.maxNodeVersion) + && Objects.equals(minNodeVersion, that.minNodeVersion); + } + + @Override + public int hashCode() { + return Objects.hash( + nodes, + dataNodes, + clusterManagerNodes, + ingestNodes, + clusterManagerNodeId, + localNodeId, + minNonClientNodeVersion, + maxNonClientNodeVersion, + maxNodeVersion, + minNodeVersion + ); } public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 80c9d42cdb617..9cc3bb21e2d12 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -45,8 +45,10 @@ import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.common.Randomness; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -79,7 +81,10 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndexRoutingTable extends AbstractDiffable implements Iterable { +public class IndexRoutingTable extends AbstractDiffable + implements + Iterable, + VerifiableWriteable { private final Index index; private final ShardShuffler shuffler; @@ -353,6 +358,11 @@ public int hashCode() { return result; } + @Override + public String toString() { + return "IndexRoutingTable{" + "shards=" + shards + ", index=" + index + '}'; + } + public static IndexRoutingTable readFrom(StreamInput in) throws IOException { Index index = new Index(in); Builder builder = new Builder(index); @@ -378,6 +388,11 @@ public void writeTo(StreamOutput out) throws IOException { } } + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + index.writeTo(out); + out.writeMapValues(shards, (stream, value) -> IndexShardRoutingTable.Builder.writeVerifiableTo(value, stream)); + } + public static Builder builder(Index index) { return new Builder(index); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index f782cfc1b060a..4cc3300676986 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.cluster.node.DiscoveryNode; @@ -62,6 +63,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -1169,6 +1171,27 @@ public static void writeToThin(IndexShardRoutingTable indexShard, StreamOutput o } } + public static void writeVerifiableTo(IndexShardRoutingTable indexShard, StreamOutput out) throws IOException { + out.writeVInt(indexShard.shardId.id()); + out.writeVInt(indexShard.shards.size()); + // Order allocated shards by allocationId + AtomicInteger assignedShardCount = new AtomicInteger(); + indexShard.shards.stream() + .filter(shardRouting -> shardRouting.allocationId() != null) + .sorted(Comparator.comparing(o -> o.allocationId().getId())) + .forEach(shardRouting -> { + try { + assignedShardCount.getAndIncrement(); + shardRouting.writeToThin(out); + } catch (IOException e) { + logger.error(() -> new ParameterizedMessage("Failed to write shard {}. Exception {}", indexShard, e)); + throw new RuntimeException("Failed to write IndexShardRoutingTable", e); + } + }); + // is primary assigned + out.writeBoolean(indexShard.primaryShard().allocationId() != null); + out.writeVInt(indexShard.shards.size() - assignedShardCount.get()); + } } @Override diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 544d4f9326fba..7128eb44bfb14 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -42,8 +42,10 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.iterable.Iterables; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexNotFoundException; @@ -70,7 +72,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class RoutingTable implements Iterable, Diffable { +public class RoutingTable implements Iterable, Diffable, VerifiableWriteable { public static final RoutingTable EMPTY_ROUTING_TABLE = builder().build(); @@ -407,6 +409,12 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeLong(version); + out.writeMapValues(indicesRouting, (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); + } + private static class RoutingTableDiff implements Diff, StringKeyDiffProvider { private final long version; @@ -426,6 +434,11 @@ private static class RoutingTableDiff implements Diff, StringKeyDi indicesRouting = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DIFF_VALUE_READER); } + @Override + public String toString() { + return "RoutingTableDiff{" + "version=" + version + ", indicesRouting=" + indicesRouting + '}'; + } + @Override public RoutingTable apply(RoutingTable part) { return new RoutingTable(version, indicesRouting.apply(part.indicesRouting)); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index f418ceab38feb..8e036209daff0 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -740,6 +740,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 8025940b35997..405e5cd784196 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -45,7 +45,8 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { // also we introduce index routing-metadata, diff and other attributes as part of manifest // required for state publication public static final int CODEC_V3 = 3; // In Codec V3, we have introduced new diff field in diff-manifest's routing_table_diff - public static final int CODEC_V4 = 4; // In Codec V4, we have removed upserts and delete field for routing table in diff manifest + public static final int CODEC_V4 = 4; // In Codec V4, we have removed upserts and delete field for routing table in diff manifest and + // added checksum of cluster state. public static final int[] CODEC_VERSIONS = { CODEC_V0, CODEC_V1, CODEC_V2, CODEC_V3, CODEC_V4 }; private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); @@ -75,6 +76,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { ); private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata"); private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); + private static final ParseField CHECKSUM = new ParseField("checksum"); private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) { return ClusterMetadataManifest.builder() @@ -117,7 +119,7 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields } private static ClusterMetadataManifest.Builder manifestV4Builder(Object[] fields) { - return manifestV3Builder(fields); + return manifestV3Builder(fields).checksum(checksum(fields)); } private static long term(Object[] fields) { @@ -222,6 +224,10 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { return (ClusterStateDiffManifest) fields[23]; } + private static ClusterStateChecksum checksum(Object[] fields) { + return (ClusterStateChecksum) fields[24]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -248,6 +254,7 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { ); private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V4; + public static final int MANIFEST_CURRENT_CODEC_VERSION = CODEC_V4; private static final Map VERSION_TO_CODEC_MAPPING; @@ -360,6 +367,13 @@ private static void declareParser(ConstructingObjectParser= CODEC_V4) { + parser.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ClusterStateChecksum.fromXContent(p), + CHECKSUM + ); + } } private final int codecVersion; @@ -387,6 +401,7 @@ private static void declareParser(ConstructingObjectParser uploadedClusterStateCustomMap; private final ClusterStateDiffManifest diffManifest; + private ClusterStateChecksum clusterStateChecksum; public List getIndices() { return indices; @@ -495,6 +510,10 @@ public List getIndicesRouting() { return indicesRouting; } + public ClusterStateChecksum getClusterStateChecksum() { + return clusterStateChecksum; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -520,7 +539,8 @@ public ClusterMetadataManifest( UploadedMetadataAttribute uploadedTransientSettingsMetadata, UploadedMetadataAttribute uploadedHashesOfConsistentSettings, Map uploadedClusterStateCustomMap, - ClusterStateDiffManifest diffManifest + ClusterStateDiffManifest diffManifest, + ClusterStateChecksum clusterStateChecksum ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -551,6 +571,7 @@ public ClusterMetadataManifest( this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>() ); + this.clusterStateChecksum = clusterStateChecksum; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -564,6 +585,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); + clusterStateChecksum = null; if (in.getVersion().onOrAfter(Version.V_2_15_0)) { this.codecVersion = in.readInt(); this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); @@ -626,6 +648,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedHashesOfConsistentSettings = null; this.uploadedClusterStateCustomMap = null; } + if (in.getVersion().onOrAfter(Version.V_2_17_0) && in.readBoolean()) { + clusterStateChecksum = new ClusterStateChecksum(in); + } } public static Builder builder() { @@ -723,6 +748,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } + if (onOrAfterCodecVersion(CODEC_V4)) { + if (getClusterStateChecksum() != null) { + builder.startObject(CHECKSUM.getPreferredName()); + getClusterStateChecksum().toXContent(builder, params); + builder.endObject(); + } + } return builder; } @@ -782,6 +814,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); } + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { + if (clusterStateChecksum != null) { + out.writeBoolean(true); + clusterStateChecksum.writeTo(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -817,7 +857,8 @@ public boolean equals(Object o) { && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) - && Objects.equals(diffManifest, that.diffManifest); + && Objects.equals(diffManifest, that.diffManifest) + && Objects.equals(clusterStateChecksum, that.clusterStateChecksum); } @Override @@ -847,7 +888,8 @@ public int hashCode() { uploadedTransientSettingsMetadata, uploadedHashesOfConsistentSettings, uploadedClusterStateCustomMap, - diffManifest + diffManifest, + clusterStateChecksum ); } @@ -912,6 +954,7 @@ public static class Builder { private UploadedMetadataAttribute hashesOfConsistentSettings; private Map clusterStateCustomMetadataMap; private ClusterStateDiffManifest diffManifest; + private ClusterStateChecksum checksum; public Builder indices(List indices) { this.indices = indices; @@ -1051,6 +1094,11 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) { return this; } + public Builder checksum(ClusterStateChecksum checksum) { + this.checksum = checksum; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); @@ -1083,6 +1131,7 @@ public Builder(ClusterMetadataManifest manifest) { this.diffManifest = manifest.diffManifest; this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; + this.checksum = manifest.clusterStateChecksum; } public ClusterMetadataManifest build() { @@ -1111,7 +1160,8 @@ public ClusterMetadataManifest build() { transientSettingsMetadata, hashesOfConsistentSettings, clusterStateCustomMetadataMap, - diffManifest + diffManifest, + checksum ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java new file mode 100644 index 0000000000000..d6739c4572d1a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -0,0 +1,485 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.jcraft.jzlib.JZlib; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * Stores checksum for all components in cluster state. This will be used to ensure cluster state is same across all nodes in the cluster. + */ +public class ClusterStateChecksum implements ToXContentFragment, Writeable { + + static final String ROUTING_TABLE_CS = "routing_table"; + static final String NODES_CS = "discovery_nodes"; + static final String BLOCKS_CS = "blocks"; + static final String CUSTOMS_CS = "customs"; + static final String COORDINATION_MD_CS = "coordination_md"; + static final String SETTINGS_MD_CS = "settings_md"; + static final String TRANSIENT_SETTINGS_MD_CS = "transient_settings_md"; + static final String TEMPLATES_MD_CS = "templates_md"; + static final String CUSTOM_MD_CS = "customs_md"; + static final String HASHES_MD_CS = "hashes_md"; + static final String INDICES_CS = "indices_md"; + private static final String CLUSTER_STATE_CS = "cluster_state"; + private static final int CHECKSUM_SIZE = 8; + private static final Logger logger = LogManager.getLogger(ClusterStateChecksum.class); + + long routingTableChecksum; + long nodesChecksum; + long blocksChecksum; + long clusterStateCustomsChecksum; + long coordinationMetadataChecksum; + long settingMetadataChecksum; + long transientSettingsMetadataChecksum; + long templatesMetadataChecksum; + long customMetadataMapChecksum; + long hashesOfConsistentSettingsChecksum; + long indicesChecksum; + long clusterStateChecksum; + + public ClusterStateChecksum(ClusterState clusterState) { + try ( + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) + ) { + clusterState.routingTable().writeVerifiableTo(checksumOut); + routingTableChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.nodes().writeVerifiableTo(checksumOut); + nodesChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.coordinationMetadata().writeVerifiableTo(checksumOut); + coordinationMetadataChecksum = checksumOut.getChecksum(); + + // Settings create sortedMap by default, so no explicit sorting required here. + checksumOut.reset(); + Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); + settingMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), checksumOut); + transientSettingsMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.metadata().templatesMetadata().writeVerifiableTo(checksumOut); + templatesMetadataChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + checksumOut.writeStringCollection(clusterState.metadata().customs().keySet()); + customMetadataMapChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); + hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + checksumOut.writeMapValues( + clusterState.metadata().indices(), + (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream) + ); + indicesChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + clusterState.blocks().writeVerifiableTo(checksumOut); + blocksChecksum = checksumOut.getChecksum(); + + checksumOut.reset(); + checksumOut.writeStringCollection(clusterState.customs().keySet()); + clusterStateCustomsChecksum = checksumOut.getChecksum(); + } catch (IOException e) { + logger.error("Failed to create checksum for cluster state.", e); + throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e); + } + createClusterStateChecksum(); + } + + private void createClusterStateChecksum() { + clusterStateChecksum = JZlib.crc32_combine(routingTableChecksum, nodesChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, blocksChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, clusterStateCustomsChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, coordinationMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, settingMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, transientSettingsMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, templatesMetadataChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, customMetadataMapChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, hashesOfConsistentSettingsChecksum, CHECKSUM_SIZE); + clusterStateChecksum = JZlib.crc32_combine(clusterStateChecksum, indicesChecksum, CHECKSUM_SIZE); + } + + public static ClusterStateChecksum.Builder builder() { + return new ClusterStateChecksum.Builder(); + } + + public ClusterStateChecksum( + long routingTableChecksum, + long nodesChecksum, + long blocksChecksum, + long clusterStateCustomsChecksum, + long coordinationMetadataChecksum, + long settingMetadataChecksum, + long transientSettingsMetadataChecksum, + long templatesMetadataChecksum, + long customMetadataMapChecksum, + long hashesOfConsistentSettingsChecksum, + long indicesChecksum, + long clusterStateChecksum + ) { + this.routingTableChecksum = routingTableChecksum; + this.nodesChecksum = nodesChecksum; + this.blocksChecksum = blocksChecksum; + this.clusterStateCustomsChecksum = clusterStateCustomsChecksum; + this.coordinationMetadataChecksum = coordinationMetadataChecksum; + this.settingMetadataChecksum = settingMetadataChecksum; + this.transientSettingsMetadataChecksum = transientSettingsMetadataChecksum; + this.templatesMetadataChecksum = templatesMetadataChecksum; + this.customMetadataMapChecksum = customMetadataMapChecksum; + this.hashesOfConsistentSettingsChecksum = hashesOfConsistentSettingsChecksum; + this.indicesChecksum = indicesChecksum; + this.clusterStateChecksum = clusterStateChecksum; + } + + public ClusterStateChecksum(StreamInput in) throws IOException { + routingTableChecksum = in.readLong(); + nodesChecksum = in.readLong(); + blocksChecksum = in.readLong(); + clusterStateCustomsChecksum = in.readLong(); + coordinationMetadataChecksum = in.readLong(); + settingMetadataChecksum = in.readLong(); + transientSettingsMetadataChecksum = in.readLong(); + templatesMetadataChecksum = in.readLong(); + customMetadataMapChecksum = in.readLong(); + hashesOfConsistentSettingsChecksum = in.readLong(); + indicesChecksum = in.readLong(); + clusterStateChecksum = in.readLong(); + } + + public static ClusterStateChecksum fromXContent(XContentParser parser) throws IOException { + ClusterStateChecksum.Builder builder = new ClusterStateChecksum.Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + switch (currentFieldName) { + case ROUTING_TABLE_CS: + builder.routingTableChecksum(parser.longValue()); + break; + case NODES_CS: + builder.nodesChecksum(parser.longValue()); + break; + case BLOCKS_CS: + builder.blocksChecksum(parser.longValue()); + break; + case CUSTOMS_CS: + builder.clusterStateCustomsChecksum(parser.longValue()); + break; + case COORDINATION_MD_CS: + builder.coordinationMetadataChecksum(parser.longValue()); + break; + case SETTINGS_MD_CS: + builder.settingMetadataChecksum(parser.longValue()); + break; + case TRANSIENT_SETTINGS_MD_CS: + builder.transientSettingsMetadataChecksum(parser.longValue()); + break; + case TEMPLATES_MD_CS: + builder.templatesMetadataChecksum(parser.longValue()); + break; + case CUSTOM_MD_CS: + builder.customMetadataMapChecksum(parser.longValue()); + break; + case HASHES_MD_CS: + builder.hashesOfConsistentSettingsChecksum(parser.longValue()); + break; + case INDICES_CS: + builder.indicesChecksum(parser.longValue()); + break; + case CLUSTER_STATE_CS: + builder.clusterStateChecksum(parser.longValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(routingTableChecksum); + out.writeLong(nodesChecksum); + out.writeLong(blocksChecksum); + out.writeLong(clusterStateCustomsChecksum); + out.writeLong(coordinationMetadataChecksum); + out.writeLong(settingMetadataChecksum); + out.writeLong(transientSettingsMetadataChecksum); + out.writeLong(templatesMetadataChecksum); + out.writeLong(customMetadataMapChecksum); + out.writeLong(hashesOfConsistentSettingsChecksum); + out.writeLong(indicesChecksum); + out.writeLong(clusterStateChecksum); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(ROUTING_TABLE_CS, routingTableChecksum); + builder.field(NODES_CS, nodesChecksum); + builder.field(BLOCKS_CS, blocksChecksum); + builder.field(CUSTOMS_CS, clusterStateCustomsChecksum); + builder.field(COORDINATION_MD_CS, coordinationMetadataChecksum); + builder.field(SETTINGS_MD_CS, settingMetadataChecksum); + builder.field(TRANSIENT_SETTINGS_MD_CS, transientSettingsMetadataChecksum); + builder.field(TEMPLATES_MD_CS, templatesMetadataChecksum); + builder.field(CUSTOM_MD_CS, customMetadataMapChecksum); + builder.field(HASHES_MD_CS, hashesOfConsistentSettingsChecksum); + builder.field(INDICES_CS, indicesChecksum); + builder.field(CLUSTER_STATE_CS, clusterStateChecksum); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterStateChecksum that = (ClusterStateChecksum) o; + return routingTableChecksum == that.routingTableChecksum + && nodesChecksum == that.nodesChecksum + && blocksChecksum == that.blocksChecksum + && clusterStateCustomsChecksum == that.clusterStateCustomsChecksum + && coordinationMetadataChecksum == that.coordinationMetadataChecksum + && settingMetadataChecksum == that.settingMetadataChecksum + && transientSettingsMetadataChecksum == that.transientSettingsMetadataChecksum + && templatesMetadataChecksum == that.templatesMetadataChecksum + && customMetadataMapChecksum == that.customMetadataMapChecksum + && hashesOfConsistentSettingsChecksum == that.hashesOfConsistentSettingsChecksum + && indicesChecksum == that.indicesChecksum + && clusterStateChecksum == that.clusterStateChecksum; + } + + @Override + public int hashCode() { + return Objects.hash( + routingTableChecksum, + nodesChecksum, + blocksChecksum, + clusterStateCustomsChecksum, + coordinationMetadataChecksum, + settingMetadataChecksum, + transientSettingsMetadataChecksum, + templatesMetadataChecksum, + customMetadataMapChecksum, + hashesOfConsistentSettingsChecksum, + indicesChecksum, + clusterStateChecksum + ); + } + + @Override + public String toString() { + return "ClusterStateChecksum{" + + "routingTableChecksum=" + + routingTableChecksum + + ", nodesChecksum=" + + nodesChecksum + + ", blocksChecksum=" + + blocksChecksum + + ", clusterStateCustomsChecksum=" + + clusterStateCustomsChecksum + + ", coordinationMetadataChecksum=" + + coordinationMetadataChecksum + + ", settingMetadataChecksum=" + + settingMetadataChecksum + + ", transientSettingsMetadataChecksum=" + + transientSettingsMetadataChecksum + + ", templatesMetadataChecksum=" + + templatesMetadataChecksum + + ", customMetadataMapChecksum=" + + customMetadataMapChecksum + + ", hashesOfConsistentSettingsChecksum=" + + hashesOfConsistentSettingsChecksum + + ", indicesChecksum=" + + indicesChecksum + + ", clusterStateChecksum=" + + clusterStateChecksum + + '}'; + } + + public List getMismatchEntities(ClusterStateChecksum otherClusterStateChecksum) { + if (this.clusterStateChecksum == otherClusterStateChecksum.clusterStateChecksum) { + logger.debug("No mismatch in checksums."); + return List.of(); + } + List mismatches = new ArrayList<>(); + addIfMismatch(this.routingTableChecksum, otherClusterStateChecksum.routingTableChecksum, ROUTING_TABLE_CS, mismatches); + addIfMismatch(this.nodesChecksum, otherClusterStateChecksum.nodesChecksum, NODES_CS, mismatches); + addIfMismatch(this.blocksChecksum, otherClusterStateChecksum.blocksChecksum, BLOCKS_CS, mismatches); + addIfMismatch(this.clusterStateCustomsChecksum, otherClusterStateChecksum.clusterStateCustomsChecksum, CUSTOMS_CS, mismatches); + addIfMismatch( + this.coordinationMetadataChecksum, + otherClusterStateChecksum.coordinationMetadataChecksum, + COORDINATION_MD_CS, + mismatches + ); + addIfMismatch(this.settingMetadataChecksum, otherClusterStateChecksum.settingMetadataChecksum, SETTINGS_MD_CS, mismatches); + addIfMismatch( + this.transientSettingsMetadataChecksum, + otherClusterStateChecksum.transientSettingsMetadataChecksum, + TRANSIENT_SETTINGS_MD_CS, + mismatches + ); + addIfMismatch(this.templatesMetadataChecksum, otherClusterStateChecksum.templatesMetadataChecksum, TEMPLATES_MD_CS, mismatches); + addIfMismatch(this.customMetadataMapChecksum, otherClusterStateChecksum.customMetadataMapChecksum, CUSTOM_MD_CS, mismatches); + addIfMismatch( + this.hashesOfConsistentSettingsChecksum, + otherClusterStateChecksum.hashesOfConsistentSettingsChecksum, + HASHES_MD_CS, + mismatches + ); + addIfMismatch(this.indicesChecksum, otherClusterStateChecksum.indicesChecksum, INDICES_CS, mismatches); + + return mismatches; + } + + private void addIfMismatch(long checksum, long otherChecksum, String entityName, List mismatches) { + if (checksum != otherChecksum) { + mismatches.add(entityName); + } + } + + /** + * Builder for ClusterStateChecksum + */ + public static class Builder { + long routingTableChecksum; + long nodesChecksum; + long blocksChecksum; + long clusterStateCustomsChecksum; + long coordinationMetadataChecksum; + long settingMetadataChecksum; + long transientSettingsMetadataChecksum; + long templatesMetadataChecksum; + long customMetadataMapChecksum; + long hashesOfConsistentSettingsChecksum; + long indicesChecksum; + long clusterStateChecksum; + + public Builder routingTableChecksum(long routingTableChecksum) { + this.routingTableChecksum = routingTableChecksum; + return this; + } + + public Builder nodesChecksum(long nodesChecksum) { + this.nodesChecksum = nodesChecksum; + return this; + } + + public Builder blocksChecksum(long blocksChecksum) { + this.blocksChecksum = blocksChecksum; + return this; + } + + public Builder clusterStateCustomsChecksum(long clusterStateCustomsChecksum) { + this.clusterStateCustomsChecksum = clusterStateCustomsChecksum; + return this; + } + + public Builder coordinationMetadataChecksum(long coordinationMetadataChecksum) { + this.coordinationMetadataChecksum = coordinationMetadataChecksum; + return this; + } + + public Builder settingMetadataChecksum(long settingMetadataChecksum) { + this.settingMetadataChecksum = settingMetadataChecksum; + return this; + } + + public Builder transientSettingsMetadataChecksum(long transientSettingsMetadataChecksum) { + this.transientSettingsMetadataChecksum = transientSettingsMetadataChecksum; + return this; + } + + public Builder templatesMetadataChecksum(long templatesMetadataChecksum) { + this.templatesMetadataChecksum = templatesMetadataChecksum; + return this; + } + + public Builder customMetadataMapChecksum(long customMetadataMapChecksum) { + this.customMetadataMapChecksum = customMetadataMapChecksum; + return this; + } + + public Builder hashesOfConsistentSettingsChecksum(long hashesOfConsistentSettingsChecksum) { + this.hashesOfConsistentSettingsChecksum = hashesOfConsistentSettingsChecksum; + return this; + } + + public Builder indicesChecksum(long indicesChecksum) { + this.indicesChecksum = indicesChecksum; + return this; + } + + public Builder clusterStateChecksum(long clusterStateChecksum) { + this.clusterStateChecksum = clusterStateChecksum; + return this; + } + + public ClusterStateChecksum build() { + return new ClusterStateChecksum( + routingTableChecksum, + nodesChecksum, + blocksChecksum, + clusterStateCustomsChecksum, + coordinationMetadataChecksum, + settingMetadataChecksum, + transientSettingsMetadataChecksum, + templatesMetadataChecksum, + customMetadataMapChecksum, + hashesOfConsistentSettingsChecksum, + indicesChecksum, + clusterStateChecksum + ); + } + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 098a36c7d42c4..88dda3953a5f9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -90,6 +90,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static org.opensearch.cluster.ClusterState.CUSTOM_VALUE_SERIALIZER; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; @@ -140,6 +141,13 @@ public class RemoteClusterStateService implements Closeable { Setting.Property.NodeScope ); + public static final Setting REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING = Setting.boolSetting( + "cluster.remote_store.state.checksum_validation.enabled", + false, + Property.Dynamic, + Property.NodeScope + ); + private TimeValue remoteStateReadTimeout; private final String nodeId; private final Supplier repositoriesService; @@ -151,6 +159,7 @@ public class RemoteClusterStateService implements Closeable { private BlobStoreTransferService blobStoreTransferService; private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; + private boolean checksumValidationEnabled; private final RemotePersistenceStats remoteStateStats; private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; @@ -197,6 +206,12 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteStateReadTimeout); + this.checksumValidationEnabled = clusterSettings.get(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING); + clusterSettings.addSettingsUpdateConsumer( + REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, + this::setChecksumValidationEnabled + ); + this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; this.indexMetadataUploadListeners = indexMetadataUploadListeners; @@ -257,6 +272,7 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat uploadedMetadataResults, previousClusterUUID, clusterStateDiffManifest, + checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, false, codecVersion ); @@ -456,6 +472,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), clusterStateDiffManifest, + checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, false, previousManifest.getCodecVersion() ); @@ -900,6 +917,7 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted(ClusterState clus uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), + checksumValidationEnabled ? previousManifest.getClusterStateChecksum() : null, true, previousManifest.getCodecVersion() ); @@ -985,6 +1003,10 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } + private void setChecksumValidationEnabled(Boolean checksumValidationEnabled) { + this.checksumValidationEnabled = checksumValidationEnabled; + } + // Package private for unit test RemoteRoutingTableService getRemoteRoutingTableService() { return this.remoteRoutingTableService; @@ -1353,6 +1375,10 @@ public ClusterState getClusterStateForManifest( false, includeEphemeral ); + + if (includeEphemeral && checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { + validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); + } } else { ClusterState state = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), @@ -1472,6 +1498,9 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); + if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { + validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); + } final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateDiffDownloadSucceeded(); remoteStateStats.stateDiffDownloadTook(durationMillis); @@ -1479,6 +1508,167 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C return clusterState; } + void validateClusterStateFromChecksum( + ClusterMetadataManifest manifest, + ClusterState clusterState, + String clusterName, + String localNodeId, + boolean isFullStateDownload + ) { + ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); + List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); + if (!failedValidation.isEmpty()) { + logger.error( + () -> new ParameterizedMessage( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", + manifest.getClusterStateChecksum(), + newClusterStateChecksum, + failedValidation + ) + ); + if (isFullStateDownload) { + throw new IllegalStateException( + "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation + ); + } + // download full cluster state and match against state created for the failing entities + ClusterState fullClusterState = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + manifest.getTransientSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + manifest.getDiscoveryNodesMetadata() != null, + manifest.getClusterBlocksMetadata() != null, + manifest.getIndicesRouting(), + manifest.getHashesOfConsistentSettings() != null, + manifest.getClusterStateCustomMap(), + false, + true + ); + for (String failedEntity : failedValidation) { + switch (failedEntity) { + case ClusterStateChecksum.ROUTING_TABLE_CS: + Diff routingTableDiff = fullClusterState.routingTable().diff(clusterState.routingTable()); + logger.error(() -> new ParameterizedMessage("Failing Diff in routing table {}", routingTableDiff)); + break; + case ClusterStateChecksum.NODES_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in discovery nodes {}", + fullClusterState.nodes().diff(clusterState.nodes()) + ) + ); + break; + case ClusterStateChecksum.BLOCKS_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in cluster blocks {}", + fullClusterState.blocks().diff(clusterState.blocks()) + ) + ); + break; + case ClusterStateChecksum.CUSTOMS_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in cluster state customs {}", + DiffableUtils.diff( + clusterState.customs(), + fullClusterState.customs(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_VALUE_SERIALIZER + ) + ) + ); + break; + case ClusterStateChecksum.COORDINATION_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in coordination md. current md {}, full state md {}", + clusterState.metadata().coordinationMetadata(), + fullClusterState.metadata().coordinationMetadata() + ) + ); + break; + case ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in transient settings md. current md {}, full state md {}", + clusterState.metadata().transientSettings(), + fullClusterState.metadata().transientSettings() + ) + ); + + break; + case ClusterStateChecksum.SETTINGS_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in settings md. current md {}, full state md {}", + clusterState.metadata().settings(), + fullClusterState.metadata().settings() + ) + ); + + break; + case ClusterStateChecksum.HASHES_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in hashes md {}", + ((DiffableStringMap) fullClusterState.metadata().hashesOfConsistentSettings()).diff( + (DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings() + ) + ) + ); + break; + case ClusterStateChecksum.TEMPLATES_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in templates md{}", + fullClusterState.metadata().templatesMetadata().diff(clusterState.metadata().templatesMetadata()) + ) + ); + break; + case ClusterStateChecksum.CUSTOM_MD_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in customs md {}", + DiffableUtils.diff( + clusterState.metadata().customs(), + fullClusterState.metadata().customs(), + DiffableUtils.getStringKeySerializer(), + Metadata.CUSTOM_VALUE_SERIALIZER + ) + ) + ); + break; + case ClusterStateChecksum.INDICES_CS: + logger.error( + () -> new ParameterizedMessage( + "Failing Diff in index md {}", + DiffableUtils.diff( + clusterState.metadata().indices(), + fullClusterState.metadata().indices(), + DiffableUtils.getStringKeySerializer() + ) + ) + ); + break; + default: + logger.error(() -> new ParameterizedMessage("Unknown failed entity {}", failedEntity)); + break; + } + } + throw new IllegalStateException( + "Cluster state checksums do not match during diff read. Validation failed for " + failedValidation + ); + } + } + /** * Fetch the previous cluster UUIDs from remote state store and return the most recent valid cluster UUID * diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 67b1b77447bed..47c847b5dc32a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; @@ -98,6 +99,7 @@ RemoteClusterStateManifestInfo uploadManifest( RemoteClusterStateUtils.UploadedMetadataResults uploadedMetadataResult, String previousClusterUUID, ClusterStateDiffManifest clusterDiffManifest, + ClusterStateChecksum clusterStateChecksum, boolean committed, int codecVersion ) { @@ -126,8 +128,10 @@ RemoteClusterStateManifestInfo uploadManifest( .metadataVersion(clusterState.metadata().version()) .transientSettingsMetadata(uploadedMetadataResult.uploadedTransientSettingsMetadata) .clusterStateCustomMetadataMap(uploadedMetadataResult.uploadedClusterStateCustomMetadataMap) - .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings); + .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings) + .checksum(clusterStateChecksum); final ClusterMetadataManifest manifest = manifestBuilder.build(); + logger.trace(() -> new ParameterizedMessage("[{}] uploading manifest", manifest)); String manifestFileName = writeMetadataManifest(clusterState.metadata().clusterUUID(), manifest); return new RemoteClusterStateManifestInfo(manifest, manifestFileName); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 9460e240e1706..999beaa4e865d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -35,6 +35,7 @@ public class RemoteClusterMetadataManifest extends AbstractClusterMetadataWritea public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; + public static final String COMMITTED = "C"; public static final String PUBLISHED = "P"; diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java new file mode 100644 index 0000000000000..839e831d38b1b --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.block; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; + +public class ClusterBlocksTests extends OpenSearchTestCase { + + public void testWriteVerifiableTo() throws Exception { + ClusterBlock clusterBlock1 = randomClusterBlock(); + ClusterBlock clusterBlock2 = randomClusterBlock(); + ClusterBlock clusterBlock3 = randomClusterBlock(); + + ClusterBlocks clusterBlocks = ClusterBlocks.builder() + .addGlobalBlock(clusterBlock1) + .addGlobalBlock(clusterBlock2) + .addGlobalBlock(clusterBlock3) + .addIndexBlock("index-1", clusterBlock1) + .addIndexBlock("index-2", clusterBlock2) + .build(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + clusterBlocks.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + ClusterBlocks result = ClusterBlocks.readFrom(in); + + assertEquals(clusterBlocks.global().size(), result.global().size()); + assertEquals(clusterBlocks.global(), result.global()); + assertEquals(clusterBlocks.indices().size(), result.indices().size()); + assertEquals(clusterBlocks.indices(), result.indices()); + + ClusterBlocks clusterBlocks2 = ClusterBlocks.builder() + .addGlobalBlock(clusterBlock3) + .addGlobalBlock(clusterBlock1) + .addGlobalBlock(clusterBlock2) + .addIndexBlock("index-2", clusterBlock2) + .addIndexBlock("index-1", clusterBlock1) + .build(); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + clusterBlocks2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java index 290479941aaa9..74c25246c2aa4 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationMetadataTests.java @@ -33,10 +33,13 @@ import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -46,8 +49,11 @@ import java.io.IOException; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -231,4 +237,29 @@ public void testXContent() throws IOException { assertThat(originalMeta, equalTo(fromXContentMeta)); } } + + public void testWriteVerifiableTo() throws IOException { + VotingConfiguration votingConfiguration = randomVotingConfig(); + Set votingTombstones = randomVotingTombstones(); + CoordinationMetadata meta1 = new CoordinationMetadata(1, votingConfiguration, votingConfiguration, votingTombstones); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + meta1.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + CoordinationMetadata result = new CoordinationMetadata(in); + + assertEquals(meta1, result); + + VotingConfiguration votingConfiguration2 = new VotingConfiguration( + (Set) votingConfiguration.getNodeIds().stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)) + ); + Set votingTombstones2 = votingTombstones.stream() + .sorted(Comparator.comparing(VotingConfigExclusion::getNodeId)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + CoordinationMetadata meta2 = new CoordinationMetadata(1, votingConfiguration2, votingConfiguration2, votingTombstones2); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + meta2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index 3b0a2fed52c2d..a92a5b41e8580 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -43,6 +43,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; @@ -173,6 +174,86 @@ public void testIndexMetadataSerialization() throws IOException { } } + public void testWriteVerifiableTo() throws IOException { + int numberOfReplicas = randomIntBetween(0, 10); + final boolean system = randomBoolean(); + Map customMap = new HashMap<>(); + customMap.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); + customMap.put(randomAlphaOfLength(10), randomAlphaOfLength(15)); + + RolloverInfo info1 = new RolloverInfo( + randomAlphaOfLength(5), + Arrays.asList( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()) + ), + randomNonNegativeLong() + ); + RolloverInfo info2 = new RolloverInfo( + randomAlphaOfLength(5), + Arrays.asList( + new MaxAgeCondition(TimeValue.timeValueMillis(randomNonNegativeLong())), + new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())), + new MaxDocsCondition(randomNonNegativeLong()) + ), + randomNonNegativeLong() + ); + + IndexMetadata metadata1 = IndexMetadata.builder("foo") + .settings( + Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 4) + .put("index.number_of_replicas", numberOfReplicas) + .build() + ) + .creationDate(randomLong()) + .primaryTerm(0, 2) + .primaryTerm(1, 3) + .setRoutingNumShards(32) + .system(system) + .putCustom("my_custom", customMap) + .putCustom("my_custom2", customMap) + .putAlias(AliasMetadata.builder("alias-1").routing("routing-1").build()) + .putAlias(AliasMetadata.builder("alias-2").routing("routing-2").build()) + .putRolloverInfo(info1) + .putRolloverInfo(info2) + .putInSyncAllocationIds(0, Set.of("1", "2", "3")) + .build(); + + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + metadata1.writeVerifiableTo(checksumOut); + + IndexMetadata metadata2 = IndexMetadata.builder(metadata1.getIndex().getName()) + .settings( + Settings.builder() + .put("index.number_of_replicas", numberOfReplicas) + .put("index.number_of_shards", 4) + .put("index.version.created", 1) + .build() + ) + .creationDate(metadata1.getCreationDate()) + .primaryTerm(1, 3) + .primaryTerm(0, 2) + .setRoutingNumShards(32) + .system(system) + .putCustom("my_custom2", customMap) + .putCustom("my_custom", customMap) + .putAlias(AliasMetadata.builder("alias-2").routing("routing-2").build()) + .putAlias(AliasMetadata.builder("alias-1").routing("routing-1").build()) + .putRolloverInfo(info2) + .putRolloverInfo(info1) + .putInSyncAllocationIds(0, Set.of("3", "1", "2")) + .build(); + + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + metadata2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testGetRoutingFactor() { Integer numShard = randomFrom(1, 2, 4, 8, 16); int routingFactor = IndexMetadata.getRoutingFactor(32, numShard); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java index 0ea2834cc3024..6b0d5bcd980d8 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexTemplateMetadataTests.java @@ -31,11 +31,14 @@ package org.opensearch.cluster.metadata; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -232,6 +235,50 @@ public void testFromToXContent() throws Exception { } } + public void testWriteVerifiableTo() throws Exception { + String templateName = randomUnicodeOfCodepointLengthBetween(1, 10); + IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder(templateName); + templateBuilder.patterns(Arrays.asList("pattern-1", "pattern-2")); + int numAlias = between(2, 5); + for (int i = 0; i < numAlias; i++) { + AliasMetadata.Builder alias = AliasMetadata.builder(randomRealisticUnicodeOfLengthBetween(1, 100)); + alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); + alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 100)); + templateBuilder.putAlias(alias); + } + templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); + templateBuilder.settings(Settings.builder().put("index.setting-2", randomTimeValue())); + templateBuilder.order(randomInt()); + templateBuilder.version(between(0, 100)); + templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}"); + + IndexTemplateMetadata template = templateBuilder.build(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + template.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + IndexTemplateMetadata result = IndexTemplateMetadata.readFrom(in); + assertEquals(result, template); + + IndexTemplateMetadata.Builder templateBuilder2 = IndexTemplateMetadata.builder(templateName); + templateBuilder2.patterns(Arrays.asList("pattern-2", "pattern-1")); + template.getAliases() + .entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .forEachOrdered(entry -> templateBuilder2.putAlias(entry.getValue())); + templateBuilder2.settings(template.settings()); + templateBuilder2.order(template.order()); + templateBuilder2.version(template.version()); + templateBuilder2.putMapping("doc", template.mappings()); + + IndexTemplateMetadata template2 = templateBuilder.build(); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + template2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testDeprecationWarningsOnMultipleMappings() throws IOException { IndexTemplateMetadata.Builder builder = IndexTemplateMetadata.builder("my-template"); builder.patterns(Arrays.asList("a", "b")); diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index c8a6fc76ce820..525a53f3e6158 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -36,6 +36,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -46,6 +47,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -128,6 +130,43 @@ public void testDiscoveryNodeSerializationKeepsHost() throws Exception { assertEquals(transportAddress.getPort(), serialized.getAddress().getPort()); } + public void testWriteVerifiableTo() throws Exception { + InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); + TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); + final Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); + Map attributes = new HashMap<>(); + attributes.put("att-1", "test-repo"); + attributes.put("att-2", "test-repo"); + DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, attributes, roles, Version.CURRENT); + + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + node.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + DiscoveryNode result = new DiscoveryNode(in); + assertEquals(result, node); + + Map attributes2 = new HashMap<>(); + attributes2.put("att-2", "test-repo"); + attributes2.put("att-1", "test-repo"); + + DiscoveryNode node2 = new DiscoveryNode( + node.getName(), + node.getId(), + node.getEphemeralId(), + node.getHostName(), + node.getHostAddress(), + transportAddress, + attributes2, + roles.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)), + Version.CURRENT + ); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + node2.writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testDiscoveryNodeRoleWithOldVersion() throws Exception { InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java index d2450859dfcd4..61b86856c9ebc 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java @@ -36,10 +36,14 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -328,6 +332,28 @@ public void testDeprecatedMasterNodeFilter() { assertThat(discoveryNodes.resolveNodes("master:false", "_all"), arrayContainingInAnyOrder(allNodes)); } + public void testWriteVerifiableTo() throws IOException { + final DiscoveryNodes discoveryNodes = buildDiscoveryNodes(); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + discoveryNodes.writeVerifiableTo(checksumOut); + StreamInput in = out.bytes().streamInput(); + DiscoveryNodes result = DiscoveryNodes.readFrom(in, discoveryNodes.getLocalNode()); + assertEquals(result, discoveryNodes); + + final DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder() + .clusterManagerNodeId(discoveryNodes.getClusterManagerNodeId()); + discoveryNodes.getNodes() + .entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .forEachOrdered(entry -> discoveryNodesBuilder.add(entry.getValue())); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + discoveryNodesBuilder.build().writeVerifiableTo(checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + private static AtomicInteger idGenerator = new AtomicInteger(); private static List randomNodes(final int numNodes) { diff --git a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java index 6bfe60980adf3..5f8dabdcd4e45 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java @@ -32,10 +32,13 @@ package org.opensearch.cluster.routing; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -77,6 +80,25 @@ public void testEquals() { assertEquals(table4, table5); } + public void testWriteVerifiableTo() throws IOException { + Index index = new Index("a", "b"); + ShardId shardId = new ShardId(index, 1); + ShardRouting shard1 = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED); + ShardRouting shard2 = TestShardRouting.newShardRouting(shardId, "node-2", false, ShardRoutingState.STARTED); + ShardRouting shard3 = TestShardRouting.newShardRouting(shardId, null, false, ShardRoutingState.UNASSIGNED); + + IndexShardRoutingTable table1 = new IndexShardRoutingTable(shardId, Arrays.asList(shard1, shard2, shard3)); + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); + IndexShardRoutingTable.Builder.writeVerifiableTo(table1, checksumOut); + + IndexShardRoutingTable table2 = new IndexShardRoutingTable(shardId, Arrays.asList(shard3, shard1, shard2)); + BytesStreamOutput out2 = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut2 = new BufferedChecksumStreamOutput(out2); + IndexShardRoutingTable.Builder.writeVerifiableTo(table2, checksumOut2); + assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); + } + public void testShardsMatchingPredicate() { ShardId shardId = new ShardId(new Index("a", UUID.randomUUID().toString()), 0); ShardRouting primary = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED); diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 97283f561d6d4..d8a67c09e442c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -714,4 +714,5 @@ public static IndexMetadata updateActiveAllocations(IndexRoutingTable indexRouti } return imdBuilder.build(); } + } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6a191d002b971..3f9aa1245cab3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -9,15 +9,24 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexGraveyard; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.StringKeyDiffProvider; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -31,6 +40,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -164,7 +174,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { .opensearchVersion(Version.CURRENT) .nodeId("B10RX1f5RJenMQvYccCgSQ") .committed(true) - .codecVersion(CODEC_V3) + .codecVersion(ClusterMetadataManifest.CODEC_V4) .indices(randomUploadedIndexMetadataList()) .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") .clusterUUIDCommitted(true) @@ -204,6 +214,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "indicesRoutingDiffPath" ) ) + .checksum(new ClusterStateChecksum(createClusterState())) .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -485,6 +496,22 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } ); } + { + // Mutate checksum + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.checksum(null); + return builder.build(); + } + ); + } } public void testClusterMetadataManifestXContentV2() throws IOException { @@ -616,15 +643,11 @@ public void testClusterMetadataManifestXContentV3() throws IOException { } } - public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOException { + public void testClusterMetadataManifestXContentV4() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - UploadedIndexMetadata uploadedIndexRoutingMetadata = new UploadedIndexMetadata( - "test-index", - "test-uuid", - "routing-path", - INDEX_ROUTING_METADATA_PREFIX - ); + final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); + ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState()); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) @@ -633,7 +656,7 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc .opensearchVersion(Version.CURRENT) .nodeId("test-node-id") .committed(false) - .codecVersion(CODEC_V2) + .codecVersion(ClusterMetadataManifest.CODEC_V4) .indices(Collections.singletonList(uploadedIndexMetadata)) .previousClusterUUID("prev-cluster-uuid") .clusterUUIDCommitted(true) @@ -658,7 +681,23 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc ) ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ) - .indicesRouting(Collections.singletonList(uploadedIndexRoutingMetadata)) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .discoveryNodesMetadata(uploadedMetadataAttribute) + .clusterBlocksMetadata(uploadedMetadataAttribute) + .transientSettingsMetadata(uploadedMetadataAttribute) + .hashesOfConsistentSettings(uploadedMetadataAttribute) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest( + new ClusterStateDiffManifest( + RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), + ClusterState.EMPTY_STATE, + CODEC_V4, + routingTableIncrementalDiff, + uploadedMetadataAttribute.getUploadedFilename() + ) + ) + .checksum(checksum) .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -671,10 +710,15 @@ public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOExc } } - public void testClusterMetadataManifestXContentV4() throws IOException { + public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); + UploadedIndexMetadata uploadedIndexRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "test-uuid", + "routing-path", + INDEX_ROUTING_METADATA_PREFIX + ); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) @@ -683,7 +727,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException { .opensearchVersion(Version.CURRENT) .nodeId("test-node-id") .committed(false) - .codecVersion(ClusterMetadataManifest.CODEC_V4) + .codecVersion(CODEC_V2) .indices(Collections.singletonList(uploadedIndexMetadata)) .previousClusterUUID("prev-cluster-uuid") .clusterUUIDCommitted(true) @@ -708,22 +752,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException { ) ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ) - .routingTableVersion(1L) - .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) - .discoveryNodesMetadata(uploadedMetadataAttribute) - .clusterBlocksMetadata(uploadedMetadataAttribute) - .transientSettingsMetadata(uploadedMetadataAttribute) - .hashesOfConsistentSettings(uploadedMetadataAttribute) - .clusterStateCustomMetadataMap(Collections.emptyMap()) - .diffManifest( - new ClusterStateDiffManifest( - RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), - ClusterState.EMPTY_STATE, - CODEC_V4, - routingTableIncrementalDiff, - uploadedMetadataAttribute.getUploadedFilename() - ) - ) + .indicesRouting(Collections.singletonList(uploadedIndexRoutingMetadata)) .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -815,4 +844,40 @@ private UploadedIndexMetadata randomlyChangingUploadedIndexMetadata(UploadedInde return uploadedIndexMetadata; } + static ClusterState createClusterState() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final Settings settings = Settings.builder().put("mock-settings", true).build(); + final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder() + .put(IndexTemplateMetadata.builder("template1").settings(idxSettings).patterns(List.of("test*")).build()) + .build(); + final RemoteClusterStateTestUtils.CustomMetadata1 customMetadata = new RemoteClusterStateTestUtils.CustomMetadata1( + "custom-metadata-1" + ); + return ClusterState.builder(ClusterName.DEFAULT) + .version(1L) + .stateUUID("state-uuid") + .metadata( + Metadata.builder() + .version(randomNonNegativeLong()) + .put(indexMetadata, true) + .clusterUUID("cluster-uuid") + .coordinationMetadata(coordinationMetadata) + .persistentSettings(settings) + .templates(templatesMetadata) + .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2")) + .putCustom(customMetadata.getWriteableName(), customMetadata) + .build() + ) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).version(1L).build()) + .build(); + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java new file mode 100644 index 0000000000000..0203e56dd2d5c --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -0,0 +1,222 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlock; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +public class ClusterStateChecksumTests extends OpenSearchTestCase { + + public void testClusterStateChecksumEmptyClusterState() { + ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE); + assertNotNull(checksum); + } + + public void testClusterStateChecksum() { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + assertNotNull(checksum); + assertTrue(checksum.routingTableChecksum != 0); + assertTrue(checksum.nodesChecksum != 0); + assertTrue(checksum.blocksChecksum != 0); + assertTrue(checksum.clusterStateCustomsChecksum != 0); + assertTrue(checksum.coordinationMetadataChecksum != 0); + assertTrue(checksum.settingMetadataChecksum != 0); + assertTrue(checksum.transientSettingsMetadataChecksum != 0); + assertTrue(checksum.templatesMetadataChecksum != 0); + assertTrue(checksum.customMetadataMapChecksum != 0); + assertTrue(checksum.hashesOfConsistentSettingsChecksum != 0); + assertTrue(checksum.indicesChecksum != 0); + assertTrue(checksum.clusterStateChecksum != 0); + } + + public void testClusterStateMatchChecksum() { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState()); + assertNotNull(checksum); + assertNotNull(newChecksum); + assertEquals(checksum.routingTableChecksum, newChecksum.routingTableChecksum); + assertEquals(checksum.nodesChecksum, newChecksum.nodesChecksum); + assertEquals(checksum.blocksChecksum, newChecksum.blocksChecksum); + assertEquals(checksum.clusterStateCustomsChecksum, newChecksum.clusterStateCustomsChecksum); + assertEquals(checksum.coordinationMetadataChecksum, newChecksum.coordinationMetadataChecksum); + assertEquals(checksum.settingMetadataChecksum, newChecksum.settingMetadataChecksum); + assertEquals(checksum.transientSettingsMetadataChecksum, newChecksum.transientSettingsMetadataChecksum); + assertEquals(checksum.templatesMetadataChecksum, newChecksum.templatesMetadataChecksum); + assertEquals(checksum.customMetadataMapChecksum, newChecksum.customMetadataMapChecksum); + assertEquals(checksum.hashesOfConsistentSettingsChecksum, newChecksum.hashesOfConsistentSettingsChecksum); + assertEquals(checksum.indicesChecksum, newChecksum.indicesChecksum); + assertEquals(checksum.clusterStateChecksum, newChecksum.clusterStateChecksum); + } + + public void testXContentConversion() throws IOException { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + checksum.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterStateChecksum parsedChecksum = ClusterStateChecksum.fromXContent(parser); + assertEquals(checksum, parsedChecksum); + } + } + + public void testSerialization() throws IOException { + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + BytesStreamOutput output = new BytesStreamOutput(); + checksum.writeTo(output); + + try (StreamInput in = output.bytes().streamInput()) { + ClusterStateChecksum deserializedChecksum = new ClusterStateChecksum(in); + assertEquals(checksum, deserializedChecksum); + } + } + + public void testGetMismatchEntities() { + ClusterState clsState1 = generateClusterState(); + ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1); + assertTrue(checksum.getMismatchEntities(checksum).isEmpty()); + + ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1); + assertTrue(checksum.getMismatchEntities(checksum2).isEmpty()); + + ClusterState clsState2 = ClusterState.builder(ClusterName.DEFAULT) + .routingTable(RoutingTable.builder().build()) + .nodes(DiscoveryNodes.builder().build()) + .blocks(ClusterBlocks.builder().build()) + .customs(Map.of()) + .metadata(Metadata.EMPTY_METADATA) + .build(); + ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2); + List mismatches = checksum.getMismatchEntities(checksum3); + assertFalse(mismatches.isEmpty()); + assertEquals(11, mismatches.size()); + assertEquals(ClusterStateChecksum.ROUTING_TABLE_CS, mismatches.get(0)); + assertEquals(ClusterStateChecksum.NODES_CS, mismatches.get(1)); + assertEquals(ClusterStateChecksum.BLOCKS_CS, mismatches.get(2)); + assertEquals(ClusterStateChecksum.CUSTOMS_CS, mismatches.get(3)); + assertEquals(ClusterStateChecksum.COORDINATION_MD_CS, mismatches.get(4)); + assertEquals(ClusterStateChecksum.SETTINGS_MD_CS, mismatches.get(5)); + assertEquals(ClusterStateChecksum.TRANSIENT_SETTINGS_MD_CS, mismatches.get(6)); + assertEquals(ClusterStateChecksum.TEMPLATES_MD_CS, mismatches.get(7)); + assertEquals(ClusterStateChecksum.CUSTOM_MD_CS, mismatches.get(8)); + assertEquals(ClusterStateChecksum.HASHES_MD_CS, mismatches.get(9)); + assertEquals(ClusterStateChecksum.INDICES_CS, mismatches.get(10)); + } + + public void testGetMismatchEntitiesUnorderedInput() { + ClusterState state1 = generateClusterState(); + DiscoveryNode node1 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9200), "node1"); + DiscoveryNode node2 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9201), "node2"); + DiscoveryNode node3 = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9202), "node3"); + + DiscoveryNodes nodes1 = DiscoveryNodes.builder().clusterManagerNodeId("test-node").add(node1).add(node2).add(node3).build(); + DiscoveryNodes nodes2 = DiscoveryNodes.builder().clusterManagerNodeId("test-node").add(node2).add(node3).build(); + nodes2 = nodes2.newNode(node1); + ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build(); + ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build(); + + ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3); + assertEquals(checksum2, checksum1); + } + + private ClusterState generateClusterState() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + final Index index2 = new Index("test-index2", "index-uuid2"); + final Settings idxSettings2 = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index2.getUUID()) + .put(IndexMetadata.INDEX_READ_ONLY_SETTING.getKey(), true) + .build(); + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) + .numberOfShards(3) + .numberOfReplicas(2) + .build(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final Settings settings = Settings.builder().put("mock-settings", true).build(); + final TemplatesMetadata templatesMetadata = TemplatesMetadata.builder() + .put(IndexTemplateMetadata.builder("template1").settings(idxSettings).patterns(List.of("test*")).build()) + .build(); + final RemoteClusterStateTestUtils.CustomMetadata1 customMetadata1 = new RemoteClusterStateTestUtils.CustomMetadata1( + "custom-metadata-1" + ); + RemoteClusterStateTestUtils.TestClusterStateCustom1 clusterStateCustom1 = new RemoteClusterStateTestUtils.TestClusterStateCustom1( + "custom-1" + ); + return ClusterState.builder(ClusterName.DEFAULT) + .version(1L) + .stateUUID("state-uuid") + .metadata( + Metadata.builder() + .version(1L) + .put(indexMetadata, true) + .clusterUUID("cluster-uuid") + .coordinationMetadata(coordinationMetadata) + .persistentSettings(settings) + .transientSettings(settings) + .templates(templatesMetadata) + .hashesOfConsistentSettings(Map.of("key1", "value1", "key2", "value2")) + .putCustom(customMetadata1.getWriteableName(), customMetadata1) + .indices(Map.of(indexMetadata.getIndex().getName(), indexMetadata, indexMetadata2.getIndex().getName(), indexMetadata2)) + .build() + ) + .nodes(DiscoveryNodes.builder().clusterManagerNodeId("test-node").build()) + .blocks( + ClusterBlocks.builder() + .addBlocks(indexMetadata) + .addGlobalBlock(new ClusterBlock(1, "block", true, true, true, RestStatus.ACCEPTED, EnumSet.of(ClusterBlockLevel.READ))) + .addGlobalBlock( + new ClusterBlock(2, "block-name", false, true, true, RestStatus.OK, EnumSet.of(ClusterBlockLevel.WRITE)) + ) + .build() + ) + .customs(Map.of(clusterStateCustom1.getWriteableName(), clusterStateCustom1)) + .routingTable(RoutingTable.builder().addAsNew(indexMetadata).addAsNew(indexMetadata2).version(1L).build()) + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 5dec516480e59..8dd35e59c43c9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -166,6 +166,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -2953,6 +2954,367 @@ private void initializeRoutingTable() { ); } + private void initializeWithChecksumEnabled() { + Settings newSettings = Settings.builder() + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) + .build(); + clusterSettings.applySettings(newSettings); + + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + newSettings, + clusterService, + () -> 0L, + threadPool, + List.of( + new RemoteIndexPathUploader( + threadPool, + newSettings, + repositoriesServiceSupplier, + clusterSettings, + DefaultRemoteStoreSettings.INSTANCE + ) + ), + writableRegistry() + ); + } + + public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + mockBlobStoreObjects(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( + clusterState, + "prev-cluster-uuid", + MANIFEST_CURRENT_CODEC_VERSION + ).getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1L) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); + } + + public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + mockBlobStoreObjects(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) + .build(); + + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indices(Collections.emptyList()) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata( + previousClusterState, + clusterState, + previousManifest + ).getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullChecksum() throws IOException { + initializeWithChecksumEnabled(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build(); + mockBlobStoreObjects(); + remoteClusterStateService.start(); + RemoteReadResult mockedResult = mock(RemoteReadResult.class); + RemoteIndexMetadataManager mockedIndexManager = mock(RemoteIndexMetadataManager.class); + RemoteGlobalMetadataManager mockedGlobalMetadataManager = mock(RemoteGlobalMetadataManager.class); + RemoteClusterStateAttributesManager mockedClusterStateAttributeManager = mock(RemoteClusterStateAttributesManager.class); + remoteClusterStateService.setRemoteIndexMetadataManager(mockedIndexManager); + remoteClusterStateService.setRemoteGlobalMetadataManager(mockedGlobalMetadataManager); + remoteClusterStateService.setRemoteClusterStateAttributesManager(mockedClusterStateAttributeManager); + ArgumentCaptor> listenerArgumentCaptor = ArgumentCaptor.forClass( + LatchedActionListener.class + ); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedIndexManager).readAsync(any(), any(), listenerArgumentCaptor.capture()); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedGlobalMetadataManager).readAsync(any(), any(), listenerArgumentCaptor.capture()); + doAnswer(invocation -> { + listenerArgumentCaptor.getValue().onResponse(mockedResult); + return null; + }).when(mockedClusterStateAttributeManager).readAsync(anyString(), any(), listenerArgumentCaptor.capture()); + when(mockedResult.getComponent()).thenReturn(COORDINATION_METADATA); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(1)).readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + verify(mockService, times(0)).validateClusterStateFromChecksum( + any(ClusterMetadataManifest.class), + any(ClusterState.class), + anyString(), + anyString(), + anyBoolean() + ); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); + } + + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + expectThrows( + IllegalStateException.class, + () -> mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true) + ); + verify(mockService, times(1)).validateClusterStateFromChecksum( + manifest, + clusterStateWrong, + ClusterName.DEFAULT.value(), + NODE_ID, + true + ); + } + + public void testGetClusterStateUsingDiffWithChecksum() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); + + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + + public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { + initializeWithChecksumEnabled(); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + + expectThrows(IllegalStateException.class, () -> mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID)); + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } @@ -3502,6 +3864,8 @@ static ClusterMetadataManifest.Builder generateClusterMetadataManifestWithAllAtt return ClusterMetadataManifest.builder() .codecVersion(CODEC_V2) .clusterUUID("cluster-uuid") + .stateVersion(1L) + .stateUUID("state-uuid") .indices(List.of(new UploadedIndexMetadata("test-index", "test-index-uuid", "test-index-file__2"))) .customMetadataMap( Map.of( @@ -3527,7 +3891,8 @@ static ClusterMetadataManifest.Builder generateClusterMetadataManifestWithAllAtt "custom_2", new UploadedMetadataAttribute("custom_2", "test-cluster-state-custom2-file__1") ) - ); + ) + .routingTableVersion(1L); } public static DiscoveryNodes nodesWithLocalNodeClusterManager() { From 7a9cb35a461384d05b49e14d028feee09593c69a Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 3 Sep 2024 18:33:39 +0530 Subject: [PATCH 023/357] Restore snapshot changes for shallow snapshot V2 (#15462) --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale Co-authored-by: Gaurav Bafna --- .../RestoreShallowSnapshotV2IT.java | 805 ++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 36 +- .../cluster/routing/RecoverySource.java | 65 +- .../opensearch/index/shard/IndexShard.java | 97 ++- .../opensearch/index/shard/StoreRecovery.java | 102 ++- .../index/translog/RemoteFsTranslog.java | 62 +- .../index/translog/TranslogConfig.java | 9 + .../transfer/TranslogTransferManager.java | 20 + .../opensearch/repositories/Repository.java | 12 + .../blobstore/BlobStoreRepository.java | 39 +- .../InternalSnapshotsInfoService.java | 26 +- .../opensearch/snapshots/RestoreService.java | 6 +- .../index/shard/IndexShardTests.java | 2 +- .../TranslogTransferManagerTests.java | 50 ++ 14 files changed, 1237 insertions(+), 94 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java new file mode 100644 index 0000000000000..c5a55f16cab2b --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -0,0 +1,805 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.recovery.RecoveryResponse; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.client.Client; +import org.opensearch.client.Requests; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.common.Nullable; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.index.Index; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.snapshots.SnapshotRestoreException; +import org.opensearch.snapshots.SnapshotState; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; +import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; +import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RestoreShallowSnapshotV2IT extends AbstractSnapshotIntegTestCase { + + private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + private Path remoteRepoPath; + + @Before + public void setup() { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } + + @After + public void teardown() { + clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + } + + @Override + protected Settings.Builder getRepositorySettings(Path location, boolean shallowCopyEnabled) { + Settings.Builder settingsBuilder = randomRepositorySettings(); + settingsBuilder.put("location", location); + if (shallowCopyEnabled) { + settingsBuilder.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + } + return settingsBuilder; + } + + private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + return settingsBuilder; + } + + private void indexDocuments(Client client, String indexName, int numOfDocs) { + indexDocuments(client, indexName, 0, numOfDocs); + } + + protected void indexDocuments(Client client, String indexName, int fromId, int toId) { + for (int i = fromId; i < toId; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + } + + private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { + for (int i = 0; i < numOfDocs; i++) { + String id = Integer.toString(i); + logger.info("checking for index " + indexName + " with docId" + id); + assertTrue("doc with id" + id + " is not present for index " + indexName, client.prepareGet(indexName, id).get().isExists()); + } + } + + public void testRestoreOperationsShallowCopyEnabled() throws Exception { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndexName1 = indexName1 + "-restored"; + String restoredIndexName2 = indexName2 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + internalCluster().startDataOnlyNode(); + logger.info("--> snapshot"); + + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>()); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + SnapshotInfo snapshotInfo2 = createSnapshot( + snapshotRepoName, + snapshotName2, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) + ); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(restoredIndexName1, restoredIndexName2); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureRed(restoredIndexName1); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); + client.admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); + ensureYellowAndNoInitializingShards(restoredIndexName1); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + public void testRemoteStoreCustomDataOnIndexCreationAndRestore() { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndexName1version1 = indexName1 + "-restored-1"; + String restoredIndexName1version2 = indexName1 + "-restored-2"; + + client(clusterManagerNode).admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED)) + .get(); + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + indexDocuments(client, indexName1, randomIntBetween(5, 10)); + ensureGreen(indexName1); + validatePathType(indexName1, RemoteStoreEnums.PathType.FIXED); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1))); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertTrue(snapshotInfo.successfulShards() > 0); + assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards()); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1version1) + .get(); + assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status()); + ensureGreen(restoredIndexName1version1); + validatePathType(restoredIndexName1version1, RemoteStoreEnums.PathType.FIXED); + + client(clusterManagerNode).admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings( + Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.HASHED_PREFIX) + ) + .get(); + + restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1version2) + .get(); + assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status()); + ensureGreen(restoredIndexName1version2); + validatePathType( + restoredIndexName1version2, + RemoteStoreEnums.PathType.HASHED_PREFIX, + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1 + ); + + // Create index with cluster setting cluster.remote_store.index.path.type as hashed_prefix. + indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings); + ensureGreen(indexName2); + validatePathType(indexName2, RemoteStoreEnums.PathType.HASHED_PREFIX, RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1); + + // Validating that custom data has not changed for indexes which were created before the cluster setting got updated + validatePathType(indexName1, RemoteStoreEnums.PathType.FIXED); + + // Create Snapshot of index 2 + String snapshotName2 = "test-restore-snapshot2"; + snapshotInfo = createSnapshot(snapshotRepoName, snapshotName2, new ArrayList<>(List.of(indexName2))); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertTrue(snapshotInfo.successfulShards() > 0); + assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards()); + + // Update cluster settings to FIXED + client(clusterManagerNode).admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED)) + .get(); + + // Close index 2 + assertAcked(client().admin().indices().prepareClose(indexName2)); + restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .get(); + assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status()); + ensureGreen(indexName2); + + // Validating that custom data has not changed for testindex2 which was created before the cluster setting got updated + validatePathType(indexName2, RemoteStoreEnums.PathType.HASHED_PREFIX, RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1); + } + + private void validatePathType(String index, RemoteStoreEnums.PathType pathType) { + validatePathType(index, pathType, null); + } + + private void validatePathType( + String index, + RemoteStoreEnums.PathType pathType, + @Nullable RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm + ) { + ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState(); + // Validate that the remote_store custom data is present in index metadata for the created index. + Map remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY); + assertNotNull(remoteCustomData); + assertEquals(pathType.name(), remoteCustomData.get(RemoteStoreEnums.PathType.NAME)); + if (Objects.nonNull(pathHashAlgorithm)) { + assertEquals(pathHashAlgorithm.name(), remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME)); + } + } + + public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndexName2 = indexName2 + "-restored"; + + boolean enableShallowCopy = randomBoolean(); + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, enableShallowCopy)); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(1, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 5; + final int numDocsInIndex2 = 6; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + internalCluster().startDataOnlyNode(); + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot( + snapshotRepoName, + snapshotName1, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) + ); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + updateRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false)); + SnapshotInfo snapshotInfo2 = createSnapshot( + snapshotRepoName, + snapshotName2, + new ArrayList<>(Arrays.asList(indexName1, indexName2)) + ); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + randomIntBetween(2, 5)); + ensureGreen(indexName1); + + assertAcked(client().admin().indices().prepareClose(indexName1)); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(false) + .setIndices(indexName2) + .setRenamePattern(indexName2) + .setRenameReplacement(restoredIndexName2) + .get(); + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + assertEquals(restoreSnapshotResponse2.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1, restoredIndexName2); + + assertRemoteSegmentsAndTranslogUploaded(restoredIndexName2); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2); + // indexing some new docs and validating + indexDocuments(client, indexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 2); + + // deleting data for restoredIndexName1 and restoring from remote store. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureRed(indexName1); + // Re-initialize client to make sure we are not using client from stopped node. + client = client(clusterManagerNode); + assertAcked(client.admin().indices().prepareClose(indexName1)); + client.admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture()); + ensureYellowAndNoInitializingShards(indexName1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + // indexing some new docs and validating + indexDocuments(client, indexName1, numDocsInIndex1 + 2, numDocsInIndex1 + 4); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, numDocsInIndex1 + 4); + } + + void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException { + Client client = client(); + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); + String path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, METADATA, translogPathFixedPrefix).buildAsString(); + Path remoteTranslogMetadataPath = Path.of(remoteRepoPath + "/" + path); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, DATA, translogPathFixedPrefix).buildAsString(); + Path remoteTranslogDataPath = Path.of(remoteRepoPath + "/" + path); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, METADATA, segmentsPathFixedPrefix).buildAsString(); + Path segmentMetadataPath = Path.of(remoteRepoPath + "/" + path); + path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, DATA, segmentsPathFixedPrefix).buildAsString(); + Path segmentDataPath = Path.of(remoteRepoPath + "/" + path); + + try ( + Stream translogMetadata = Files.list(remoteTranslogMetadataPath); + Stream translogData = Files.list(remoteTranslogDataPath); + Stream segmentMetadata = Files.list(segmentMetadataPath); + Stream segmentData = Files.list(segmentDataPath); + + ) { + assertTrue(translogData.count() > 0); + assertTrue(translogMetadata.count() > 0); + assertTrue(segmentMetadata.count() > 0); + assertTrue(segmentData.count() > 0); + } + + } + + public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, ExecutionException, InterruptedException { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + final int numDocsInIndex1 = randomIntBetween(20, 30); + indexDocuments(client(), indexName1, numDocsInIndex1); + flushAndRefresh(indexName1); + ensureGreen(indexName1); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertFalse(indexExists(indexName1)); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + + assertRemoteSegmentsAndTranslogUploaded(indexName1); + + // Clear the local data before stopping the node. This will make sure that remote translog is empty. + IndexShard indexShard = getIndexShard(primaryNodeName(indexName1), indexName1); + try (Stream files = Files.list(indexShard.shardPath().resolveTranslog())) { + IOUtils.deleteFilesIgnoringExceptions(files.collect(Collectors.toList())); + } + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1))); + + ensureRed(indexName1); + + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(false), PlainActionFuture.newFuture()); + + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + } + + private IndexShard getIndexShard(String node, String indexName) { + final Index index = resolveIndex(indexName); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(index); + assertNotNull(indexService); + final Optional shardId = indexService.shardIds().stream().findFirst(); + return shardId.map(indexService::getShard).orElse(null); + } + + public void testRestoreShallowSnapshotRepository() throws ExecutionException, InterruptedException { + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + String[] pathTokens = absolutePath1.toString().split("/"); + String basePath = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location = PathUtils.get(String.join("/", pathTokens)); + pathTokens = absolutePath2.toString().split("/"); + String basePath2 = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location2 = PathUtils.get(String.join("/", pathTokens)); + logger.info("Path 1 [{}]", absolutePath1); + logger.info("Path 2 [{}]", absolutePath2); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true)); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s") + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + createIndex(indexName1, indexSettings); + + int numDocsInIndex1 = randomIntBetween(2, 5); + indexDocuments(client, indexName1, numDocsInIndex1); + + ensureGreen(indexName1); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(List.of(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + client().admin().indices().close(Requests.closeIndexRequest(indexName1)).get(); + createRepository(remoteStoreRepoNameUpdated, "fs", remoteRepoPath); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated) + .get(); + + assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + + // ensure recovery details are non-zero + RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(restoredIndexName1).execute().actionGet(); + assertEquals(1, recoveryResponse.getTotalShards()); + assertEquals(1, recoveryResponse.getSuccessfulShards()); + assertEquals(0, recoveryResponse.getFailedShards()); + assertEquals(1, recoveryResponse.shardRecoveryStates().size()); + assertTrue(recoveryResponse.shardRecoveryStates().containsKey(restoredIndexName1)); + assertEquals(1, recoveryResponse.shardRecoveryStates().get(restoredIndexName1).size()); + + RecoveryState recoveryState = recoveryResponse.shardRecoveryStates().get(restoredIndexName1).get(0); + assertEquals(RecoveryState.Stage.DONE, recoveryState.getStage()); + assertEquals(0, recoveryState.getShardId().getId()); + assertTrue(recoveryState.getPrimary()); + assertEquals(RecoverySource.Type.SNAPSHOT, recoveryState.getRecoverySource().getType()); + assertThat(recoveryState.getIndex().time(), greaterThanOrEqualTo(0L)); + + // ensure populated file details + assertTrue(recoveryState.getIndex().totalFileCount() > 0); + assertTrue(recoveryState.getIndex().totalRecoverFiles() > 0); + assertTrue(recoveryState.getIndex().recoveredFileCount() > 0); + assertThat(recoveryState.getIndex().recoveredFilesPercent(), greaterThanOrEqualTo(0.0f)); + assertThat(recoveryState.getIndex().recoveredFilesPercent(), lessThanOrEqualTo(100.0f)); + assertFalse(recoveryState.getIndex().fileDetails().isEmpty()); + + // ensure populated bytes details + assertTrue(recoveryState.getIndex().recoveredBytes() > 0L); + assertTrue(recoveryState.getIndex().totalBytes() > 0L); + assertTrue(recoveryState.getIndex().totalRecoverBytes() > 0L); + assertThat(recoveryState.getIndex().recoveredBytesPercent(), greaterThanOrEqualTo(0.0f)); + assertThat(recoveryState.getIndex().recoveredBytesPercent(), lessThanOrEqualTo(100.0f)); + + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionException, InterruptedException { + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String remoteStoreRepoNameUpdated = "test-rs-repo-updated" + TEST_REMOTE_STORE_REPO_SUFFIX; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + Path absolutePath2 = randomRepoPath().toAbsolutePath(); + String[] pathTokens = absolutePath1.toString().split("/"); + String basePath = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location = PathUtils.get(String.join("/", pathTokens)); + pathTokens = absolutePath2.toString().split("/"); + String basePath2 = pathTokens[pathTokens.length - 1]; + Arrays.copyOf(pathTokens, pathTokens.length - 1); + Path location2 = PathUtils.get(String.join("/", pathTokens)); + logger.info("Path 1 [{}]", absolutePath1); + logger.info("Path 2 [{}]", absolutePath2); + String restoredIndexName1 = indexName1 + "-restored"; + + createRepository(snapshotRepoName, "fs", getRepositorySettings(location, basePath, true)); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + createIndex(indexName1, indexSettings); + + int numDocsInIndex1 = randomIntBetween(2, 5); + indexDocuments(client, indexName1, numDocsInIndex1); + + ensureGreen(indexName1); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(List.of(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + int extraNumDocsInIndex1 = randomIntBetween(20, 50); + indexDocuments(client, indexName1, extraNumDocsInIndex1); + refresh(indexName1); + + client().admin().indices().close(Requests.closeIndexRequest(indexName1)).get(); + createRepository(remoteStoreRepoNameUpdated, "fs", remoteRepoPath); + RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .setSourceRemoteStoreRepository(remoteStoreRepoNameUpdated) + .get(); + + assertTrue(restoreSnapshotResponse2.getRestoreInfo().failedShards() == 0); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + + // indexing some new docs and validating + indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2); + ensureGreen(restoredIndexName1); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2); + } + + public void testInvalidRestoreRequestScenarios() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + String index = "test-index"; + String snapshotRepo = "test-restore-snapshot-repo"; + String newRemoteStoreRepo = "test-new-rs-repo"; + String snapshotName1 = "test-restore-snapshot1"; + String snapshotName2 = "test-restore-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + String restoredIndex = index + "-restored"; + + createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true)); + + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(index, indexSettings); + + final int numDocsInIndex = 5; + indexDocuments(client, index, numDocsInIndex); + ensureGreen(index); + + internalCluster().startDataOnlyNode(); + logger.info("--> snapshot"); + + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index))); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + updateRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, false)); + SnapshotInfo snapshotInfo2 = createSnapshot(snapshotRepo, snapshotName2, new ArrayList<>(List.of(index))); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + + DeleteResponse deleteResponse = client().prepareDelete(index, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + indexDocuments(client, index, numDocsInIndex, numDocsInIndex + randomIntBetween(2, 5)); + ensureGreen(index); + + // try index restore with remote store disabled + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(SETTING_REMOTE_STORE_ENABLED) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore")); + + // try index restore with remote store repository modified + Settings remoteStoreIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); + + // try index restore with remote store repository and translog store repository disabled + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings( + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + ) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 492ef86bb7843..409c48cabad35 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -121,6 +121,8 @@ private static StorageType fromString(String string) { private StorageType storageType = StorageType.LOCAL; @Nullable private String sourceRemoteStoreRepository = null; + @Nullable + private String sourceRemoteTranslogRepository = null; @Nullable // if any snapshot UUID will do private String snapshotUuid; @@ -159,6 +161,9 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_10_0)) { sourceRemoteStoreRepository = in.readOptionalString(); } + if (in.getVersion().onOrAfter(Version.CURRENT)) { + sourceRemoteTranslogRepository = in.readOptionalString(); + } } @Override @@ -183,6 +188,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_10_0)) { out.writeOptionalString(sourceRemoteStoreRepository); } + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalString(sourceRemoteTranslogRepository); + } } @Override @@ -545,6 +553,16 @@ public RestoreSnapshotRequest setSourceRemoteStoreRepository(String sourceRemote return this; } + /** + * Sets Source Remote Translog Repository for all the restored indices + * + * @param sourceRemoteTranslogRepository name of the remote translog repository that should be used for all restored indices. + */ + public RestoreSnapshotRequest setSourceRemoteTranslogRepository(String sourceRemoteTranslogRepository) { + this.sourceRemoteTranslogRepository = sourceRemoteTranslogRepository; + return this; + } + /** * Returns Source Remote Store Repository for all the restored indices * @@ -554,6 +572,15 @@ public String getSourceRemoteStoreRepository() { return sourceRemoteStoreRepository; } + /** + * Returns Source Remote Translog Repository for all the restored indices + * + * @return source Remote Translog Repository + */ + public String getSourceRemoteTranslogRepository() { + return sourceRemoteTranslogRepository; + } + /** * Parses restore definition * @@ -673,6 +700,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (sourceRemoteStoreRepository != null) { builder.field("source_remote_store_repository", sourceRemoteStoreRepository); } + if (sourceRemoteTranslogRepository != null) { + builder.field("source_remote_translog_repository", sourceRemoteTranslogRepository); + } builder.endObject(); return builder; } @@ -701,7 +731,8 @@ public boolean equals(Object o) { && Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings) && Objects.equals(snapshotUuid, that.snapshotUuid) && Objects.equals(storageType, that.storageType) - && Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository); + && Objects.equals(sourceRemoteStoreRepository, that.sourceRemoteStoreRepository) + && Objects.equals(sourceRemoteTranslogRepository, that.sourceRemoteTranslogRepository); return equals; } @@ -721,7 +752,8 @@ public int hashCode() { indexSettings, snapshotUuid, storageType, - sourceRemoteStoreRepository + sourceRemoteStoreRepository, + sourceRemoteTranslogRepository ); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index 8d407c6aff5b6..43e195ed47553 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -48,6 +48,8 @@ import java.io.IOException; import java.util.Objects; +import static org.opensearch.Version.CURRENT; + /** * Represents the recovery source of a shard. Available recovery types are: *

@@ -264,6 +266,9 @@ public static class SnapshotRecoverySource extends RecoverySource { private final boolean isSearchableSnapshot; private final boolean remoteStoreIndexShallowCopy; private final String sourceRemoteStoreRepository; + private final String sourceRemoteTranslogRepository; + + private final long pinnedTimestamp; public SnapshotRecoverySource(String restoreUUID, Snapshot snapshot, Version version, IndexId indexId) { this(restoreUUID, snapshot, version, indexId, false, false, null); @@ -277,6 +282,30 @@ public SnapshotRecoverySource( boolean isSearchableSnapshot, boolean remoteStoreIndexShallowCopy, @Nullable String sourceRemoteStoreRepository + ) { + this( + restoreUUID, + snapshot, + version, + indexId, + isSearchableSnapshot, + remoteStoreIndexShallowCopy, + sourceRemoteStoreRepository, + null, + 0L + ); + } + + public SnapshotRecoverySource( + String restoreUUID, + Snapshot snapshot, + Version version, + IndexId indexId, + boolean isSearchableSnapshot, + boolean remoteStoreIndexShallowCopy, + @Nullable String sourceRemoteStoreRepository, + @Nullable String sourceRemoteTranslogRepository, + long pinnedTimestamp ) { this.restoreUUID = restoreUUID; this.snapshot = Objects.requireNonNull(snapshot); @@ -285,6 +314,8 @@ public SnapshotRecoverySource( this.isSearchableSnapshot = isSearchableSnapshot; this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy; this.sourceRemoteStoreRepository = sourceRemoteStoreRepository; + this.sourceRemoteTranslogRepository = sourceRemoteTranslogRepository; + this.pinnedTimestamp = pinnedTimestamp; } SnapshotRecoverySource(StreamInput in) throws IOException { @@ -304,6 +335,13 @@ public SnapshotRecoverySource( remoteStoreIndexShallowCopy = false; sourceRemoteStoreRepository = null; } + if (in.getVersion().onOrAfter(CURRENT)) { + sourceRemoteTranslogRepository = in.readOptionalString(); + pinnedTimestamp = in.readLong(); + } else { + sourceRemoteTranslogRepository = null; + pinnedTimestamp = 0L; + } } public String restoreUUID() { @@ -336,10 +374,18 @@ public String sourceRemoteStoreRepository() { return sourceRemoteStoreRepository; } + public String sourceRemoteTranslogRepository() { + return sourceRemoteTranslogRepository; + } + public boolean remoteStoreIndexShallowCopy() { return remoteStoreIndexShallowCopy; } + public long pinnedTimestamp() { + return pinnedTimestamp; + } + @Override protected void writeAdditionalFields(StreamOutput out) throws IOException { out.writeString(restoreUUID); @@ -353,6 +399,10 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException { out.writeBoolean(remoteStoreIndexShallowCopy); out.writeOptionalString(sourceRemoteStoreRepository); } + if (out.getVersion().onOrAfter(CURRENT)) { + out.writeOptionalString(sourceRemoteTranslogRepository); + out.writeLong(pinnedTimestamp); + } } @Override @@ -369,7 +419,8 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param .field("restoreUUID", restoreUUID) .field("isSearchableSnapshot", isSearchableSnapshot) .field("remoteStoreIndexShallowCopy", remoteStoreIndexShallowCopy) - .field("sourceRemoteStoreRepository", sourceRemoteStoreRepository); + .field("sourceRemoteStoreRepository", sourceRemoteStoreRepository) + .field("sourceRemoteTranslogRepository", sourceRemoteTranslogRepository); } @Override @@ -394,8 +445,11 @@ public boolean equals(Object o) { && isSearchableSnapshot == that.isSearchableSnapshot && remoteStoreIndexShallowCopy == that.remoteStoreIndexShallowCopy && sourceRemoteStoreRepository != null - ? sourceRemoteStoreRepository.equals(that.sourceRemoteStoreRepository) - : that.sourceRemoteStoreRepository == null; + ? sourceRemoteStoreRepository.equals(that.sourceRemoteStoreRepository) + : that.sourceRemoteStoreRepository == null && sourceRemoteTranslogRepository != null + ? sourceRemoteTranslogRepository.equals(that.sourceRemoteTranslogRepository) + : that.sourceRemoteTranslogRepository == null && pinnedTimestamp == that.pinnedTimestamp; + } @Override @@ -407,10 +461,11 @@ public int hashCode() { version, isSearchableSnapshot, remoteStoreIndexShallowCopy, - sourceRemoteStoreRepository + sourceRemoteStoreRepository, + sourceRemoteTranslogRepository, + pinnedTimestamp ); } - } /** diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 354ecd72b4b08..6e12e4ed3da1a 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -150,6 +150,7 @@ import org.opensearch.index.recovery.RecoveryStats; import org.opensearch.index.refresh.RefreshStats; import org.opensearch.index.remote.RemoteSegmentStats; +import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.search.stats.ShardSearchStats; @@ -2479,6 +2480,10 @@ private void loadGlobalCheckpointToReplicationTracker() throws IOException { * Operations from the translog will be replayed to bring lucene up to date. **/ public void openEngineAndRecoverFromTranslog() throws IOException { + openEngineAndRecoverFromTranslog(true); + } + + public void openEngineAndRecoverFromTranslog(boolean syncFromRemote) throws IOException { recoveryState.validateCurrentStage(RecoveryState.Stage.INDEX); maybeCheckIndex(); recoveryState.setStage(RecoveryState.Stage.TRANSLOG); @@ -2499,7 +2504,16 @@ public void openEngineAndRecoverFromTranslog() throws IOException { loadGlobalCheckpointToReplicationTracker(); } - innerOpenEngineAndTranslog(replicationTracker); + if (isSnapshotV2Restore()) { + translogConfig.setDownloadRemoteTranslogOnInit(false); + } + + innerOpenEngineAndTranslog(replicationTracker, syncFromRemote); + + if (isSnapshotV2Restore()) { + translogConfig.setDownloadRemoteTranslogOnInit(true); + } + getEngine().translogManager() .recoverFromTranslog(translogRecoveryRunner, getEngine().getProcessedLocalCheckpoint(), Long.MAX_VALUE); } @@ -2561,7 +2575,7 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, b if (shardRouting.primary()) { if (syncFromRemote) { syncRemoteTranslogAndUpdateGlobalCheckpoint(); - } else { + } else if (isSnapshotV2Restore() == false) { // we will enter this block when we do not want to recover from remote translog. // currently only during snapshot restore, we are coming into this block. // here, as while initiliazing remote translog we cannot skip downloading translog files, @@ -2607,6 +2621,11 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, b recoveryState.validateCurrentStage(RecoveryState.Stage.TRANSLOG); } + private boolean isSnapshotV2Restore() { + return routingEntry().recoverySource().getType() == RecoverySource.Type.SNAPSHOT + && ((SnapshotRecoverySource) routingEntry().recoverySource()).pinnedTimestamp() > 0; + } + private boolean assertSequenceNumbersInCommit() throws IOException { final Map userData = fetchUserData(); assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint"; @@ -2892,14 +2911,26 @@ public void restoreFromSnapshotAndRemoteStore( assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.SNAPSHOT : "invalid recovery type: " + recoveryState.getRecoverySource(); StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); - storeRecovery.recoverFromSnapshotAndRemoteStore( - this, - repository, - repositoriesService, - listener, - remoteStoreSettings.getSegmentsPathFixedPrefix(), - threadPool - ); + SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) recoveryState().getRecoverySource(); + if (recoverySource.pinnedTimestamp() != 0) { + storeRecovery.recoverShallowSnapshotV2( + this, + repository, + repositoriesService, + listener, + remoteStoreSettings.getSegmentsPathFixedPrefix(), + threadPool + ); + } else { + storeRecovery.recoverFromSnapshotAndRemoteStore( + this, + repository, + repositoriesService, + listener, + remoteStoreSettings.getSegmentsPathFixedPrefix(), + threadPool + ); + } } catch (Exception e) { listener.onFailure(e); } @@ -5007,16 +5038,33 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException { TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting); assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory; Repository repository = ((RemoteBlobStoreInternalTranslogFactory) translogFactory).getRepository(); + syncTranslogFilesFromGivenRemoteTranslog( + repository, + shardId, + indexSettings.getRemoteStorePathStrategy(), + indexSettings().isTranslogMetadataEnabled(), + 0 + ); + } + + public void syncTranslogFilesFromGivenRemoteTranslog( + Repository repository, + ShardId shardId, + RemoteStorePathStrategy remoteStorePathStrategy, + boolean isTranslogMetadataEnabled, + long timestamp + ) throws IOException { RemoteFsTranslog.download( repository, shardId, getThreadPool(), shardPath().resolveTranslog(), - indexSettings.getRemoteStorePathStrategy(), + remoteStorePathStrategy, remoteStoreSettings, logger, shouldSeedRemoteStore(), - indexSettings().isTranslogMetadataEnabled() + isTranslogMetadataEnabled, + timestamp ); } @@ -5105,15 +5153,13 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, final Runn * Downloads segments from given remote segment store for a specific commit. * @param overrideLocal flag to override local segment files with those in remote store * @param sourceRemoteDirectory RemoteSegmentDirectory Instance from which we need to sync segments - * @param primaryTerm Primary Term for shard at the time of commit operation for which we are syncing segments - * @param commitGeneration commit generation at the time of commit operation for which we are syncing segments * @throws IOException if exception occurs while reading segments from remote store */ public void syncSegmentsFromGivenRemoteSegmentStore( boolean overrideLocal, RemoteSegmentStoreDirectory sourceRemoteDirectory, - long primaryTerm, - long commitGeneration + RemoteSegmentMetadata remoteSegmentMetadata, + boolean pinnedTimestamp ) throws IOException { logger.trace("Downloading segments from given remote segment store"); RemoteSegmentStoreDirectory remoteDirectory = null; @@ -5149,12 +5195,29 @@ public void syncSegmentsFromGivenRemoteSegmentStore( overrideLocal, () -> {} ); - if (segmentsNFile != null) { + if (pinnedTimestamp) { + final SegmentInfos infosSnapshot = store.buildSegmentInfos( + remoteSegmentMetadata.getSegmentInfosBytes(), + remoteSegmentMetadata.getGeneration() + ); + long processedLocalCheckpoint = Long.parseLong(infosSnapshot.getUserData().get(LOCAL_CHECKPOINT_KEY)); + // delete any other commits, we want to start the engine only from a new commit made with the downloaded infos bytes. + // Extra segments will be wiped on engine open. + for (String file : List.of(store.directory().listAll())) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { + store.deleteQuiet(file); + } + } + assert Arrays.stream(store.directory().listAll()).filter(f -> f.startsWith(IndexFileNames.SEGMENTS)).findAny().isEmpty() + : "There should not be any segments file in the dir"; + store.commitSegmentInfos(infosSnapshot, processedLocalCheckpoint, processedLocalCheckpoint); + } else if (segmentsNFile != null) { try ( ChecksumIndexInput indexInput = new BufferedChecksumIndexInput( storeDirectory.openInput(segmentsNFile, IOContext.DEFAULT) ) ) { + long commitGeneration = SegmentInfos.generationFromSegmentsFileName(segmentsNFile); SegmentInfos infosSnapshot = SegmentInfos.readCommit(store.directory(), indexInput, commitGeneration); long processedLocalCheckpoint = Long.parseLong(infosSnapshot.getUserData().get(LOCAL_CHECKPOINT_KEY)); if (remoteStore != null) { diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 8649da936d81c..6933e4e161dd1 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -58,12 +58,15 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineException; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.remote.RemoteStorePathStrategy; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; +import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.translog.Checkpoint; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogHeader; @@ -72,6 +75,7 @@ import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryData; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -407,14 +411,14 @@ void recoverFromSnapshotAndRemoteStore( shardId, shallowCopyShardMetadata.getRemoteStorePathStrategy() ); - sourceRemoteDirectory.initializeToSpecificCommit( + RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificCommit( primaryTerm, commitGeneration, recoverySource.snapshot().getSnapshotId().getUUID() ); - indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, primaryTerm, commitGeneration); + indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, remoteSegmentMetadata, false); final Store store = indexShard.store(); - if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) { + if (indexShard.indexSettings.isRemoteStoreEnabled() == false) { bootstrap(indexShard, store); } else { bootstrapForSnapshot(indexShard, store); @@ -443,6 +447,98 @@ void recoverFromSnapshotAndRemoteStore( } } + void recoverShallowSnapshotV2( + final IndexShard indexShard, + Repository repository, + RepositoriesService repositoriesService, + ActionListener listener, + String segmentsPathFixedPrefix, + ThreadPool threadPool + ) { + try { + if (canRecover(indexShard)) { + indexShard.preRecovery(); + RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType(); + assert recoveryType == RecoverySource.Type.SNAPSHOT : "expected snapshot recovery type: " + recoveryType; + SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource(); + indexShard.prepareForIndexRecovery(); + + assert recoverySource.pinnedTimestamp() != 0; + final StepListener repositoryDataListener = new StepListener<>(); + repository.getRepositoryData(repositoryDataListener); + repositoryDataListener.whenComplete(repositoryData -> { + IndexId indexId = repositoryData.resolveIndexId(recoverySource.index().getName()); + IndexMetadata prevIndexMetadata = repository.getSnapshotIndexMetaData( + repositoryData, + recoverySource.snapshot().getSnapshotId(), + indexId + ); + RemoteSegmentStoreDirectoryFactory directoryFactory = new RemoteSegmentStoreDirectoryFactory( + () -> repositoriesService, + threadPool, + segmentsPathFixedPrefix + ); + String remoteSegmentStoreRepository = ((SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource()) + .sourceRemoteStoreRepository(); + if (remoteSegmentStoreRepository == null) { + remoteSegmentStoreRepository = IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get( + prevIndexMetadata.getSettings() + ); + } + RemoteStorePathStrategy remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy(prevIndexMetadata); + RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( + remoteSegmentStoreRepository, + prevIndexMetadata.getIndexUUID(), + shardId, + remoteStorePathStrategy + ); + RemoteSegmentMetadata remoteSegmentMetadata = sourceRemoteDirectory.initializeToSpecificTimestamp( + recoverySource.pinnedTimestamp() + ); + + String remoteTranslogRepository = ((SnapshotRecoverySource) indexShard.recoveryState().getRecoverySource()) + .sourceRemoteTranslogRepository(); + if (remoteTranslogRepository == null) { + remoteTranslogRepository = IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get( + prevIndexMetadata.getSettings() + ); + } + + indexShard.syncSegmentsFromGivenRemoteSegmentStore(true, sourceRemoteDirectory, remoteSegmentMetadata, true); + indexShard.syncTranslogFilesFromGivenRemoteTranslog( + repositoriesService.repository(remoteTranslogRepository), + new ShardId(prevIndexMetadata.getIndex(), shardId.id()), + remoteStorePathStrategy, + RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata), + recoverySource.pinnedTimestamp() + ); + + assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; + writeEmptyRetentionLeasesFile(indexShard); + indexShard.recoveryState().getIndex().setFileDetailsComplete(); + indexShard.openEngineAndRecoverFromTranslog(false); + indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm()); + indexShard.finalizeRecovery(); + if (indexShard.isRemoteTranslogEnabled() && indexShard.shardRouting.primary()) { + indexShard.waitForRemoteStoreSync(); + } + indexShard.postRecovery("post recovery from remote_store"); + SegmentInfos committedSegmentInfos = indexShard.store().readLastCommittedSegmentsInfo(); + try { + indexShard.getEngine() + .translogManager() + .setMinSeqNoToKeep(Long.parseLong(committedSegmentInfos.getUserData().get(SequenceNumbers.MAX_SEQ_NO)) + 1); + } catch (IllegalArgumentException e) { + logger.warn("MinSeqNoToKeep is already past the maxSeqNo from commited segment infos"); + } + listener.onResponse(true); + }, listener::onFailure); + } + } catch (Exception e) { + listener.onFailure(e); + } + } + private boolean canRecover(IndexShard indexShard) { if (indexShard.state() == IndexShardState.CLOSED) { // got closed on us, just ignore this recovery diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 66eea2a51b36a..16ebac0a44139 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -123,36 +123,6 @@ public RemoteFsTranslog( BooleanSupplier startedPrimarySupplier, RemoteTranslogTransferTracker remoteTranslogTransferTracker, RemoteStoreSettings remoteStoreSettings - ) throws IOException { - this( - config, - translogUUID, - deletionPolicy, - globalCheckpointSupplier, - primaryTermSupplier, - persistedSequenceNumberConsumer, - blobStoreRepository, - threadPool, - startedPrimarySupplier, - remoteTranslogTransferTracker, - remoteStoreSettings, - 0 - ); - } - - public RemoteFsTranslog( - TranslogConfig config, - String translogUUID, - TranslogDeletionPolicy deletionPolicy, - LongSupplier globalCheckpointSupplier, - LongSupplier primaryTermSupplier, - LongConsumer persistedSequenceNumberConsumer, - BlobStoreRepository blobStoreRepository, - ThreadPool threadPool, - BooleanSupplier startedPrimarySupplier, - RemoteTranslogTransferTracker remoteTranslogTransferTracker, - RemoteStoreSettings remoteStoreSettings, - long timestamp ) throws IOException { super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer); logger = Loggers.getLogger(getClass(), shardId); @@ -173,7 +143,9 @@ public RemoteFsTranslog( isTranslogMetadataEnabled ); try { - download(translogTransferManager, location, logger, config.shouldSeedRemote(), timestamp); + if (config.downloadRemoteTranslogOnInit()) { + download(translogTransferManager, location, logger, config.shouldSeedRemote(), 0); + } Checkpoint checkpoint = readCheckpoint(location); logger.info("Downloaded data from remote translog till maxSeqNo = {}", checkpoint.maxSeqNo); this.readers.addAll(recoverFromFiles(checkpoint)); @@ -182,6 +154,9 @@ public RemoteFsTranslog( logger.error(errorMsg); throw new IllegalStateException(errorMsg); } + if (config.downloadRemoteTranslogOnInit() == false) { + translogTransferManager.populateFileTrackerWithLocalState(this.readers); + } boolean success = false; current = null; try { @@ -214,31 +189,6 @@ RemoteTranslogTransferTracker getRemoteTranslogTracker() { return remoteTranslogTransferTracker; } - public static void download( - Repository repository, - ShardId shardId, - ThreadPool threadPool, - Path location, - RemoteStorePathStrategy pathStrategy, - RemoteStoreSettings remoteStoreSettings, - Logger logger, - boolean seedRemote, - boolean isTranslogMetadataEnabled - ) throws IOException { - download( - repository, - shardId, - threadPool, - location, - pathStrategy, - remoteStoreSettings, - logger, - seedRemote, - isTranslogMetadataEnabled, - 0 - ); - } - public static void download( Repository repository, ShardId shardId, diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java index f720f041b287c..52e20d9838fca 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java @@ -60,6 +60,7 @@ public final class TranslogConfig { private final ByteSizeValue bufferSize; private final String nodeId; private final boolean seedRemote; + private boolean downloadRemoteTranslogOnInit = true; /** * Creates a new TranslogConfig instance @@ -140,4 +141,12 @@ public String getNodeId() { public boolean shouldSeedRemote() { return seedRemote; } + + public boolean downloadRemoteTranslogOnInit() { + return downloadRemoteTranslogOnInit; + } + + public void setDownloadRemoteTranslogOnInit(boolean downloadRemoteTranslogOnInit) { + this.downloadRemoteTranslogOnInit = downloadRemoteTranslogOnInit; + } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 527fb0123d800..56a9aa6447dec 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -28,6 +28,7 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.TranslogReader; import org.opensearch.index.translog.transfer.listener.TranslogTransferListener; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.threadpool.ThreadPool; @@ -712,4 +713,23 @@ public void onFailure(Exception e) { public int getMaxRemoteTranslogReadersSettings() { return this.remoteStoreSettings.getMaxRemoteTranslogReaders(); } + + public void populateFileTrackerWithLocalState(List readers) { + if (readers == null) { + return; + } + for (TranslogReader reader : readers) { + long generation = reader.getGeneration(); + String tlogFilename = Translog.getFilename(generation); + fileTransferTracker.add(tlogFilename, true); + if (isTranslogMetadataEnabled) { + String ckpFilename = Translog.getCommitCheckpointFileName(generation); + fileTransferTracker.add(ckpFilename, true); + } + } + } + + protected FileTransferTracker getFileTransferTracker() { + return fileTransferTracker; + } } diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 136e9fb26c3dc..4338c42e20b4b 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -418,6 +418,18 @@ default RemoteStoreShardShallowCopySnapshot getRemoteStoreShallowCopyShardMetada */ IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId); + /** + * Retrieve shard snapshot status for the stored snapshot + * + * @param snapshotInfo snapshot info + * @param indexId the snapshotted index id for the shard to get status for + * @param shardId shard id + * @return snapshot status + */ + default IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotInfo snapshotInfo, IndexId indexId, ShardId shardId) { + return getShardSnapshotStatus(snapshotInfo.snapshotId(), indexId, shardId); + } + /** * Update the repository with the incoming cluster state. This method is invoked from {@link RepositoriesService#applyClusterState} and * thus the same semantics as with {@link org.opensearch.cluster.ClusterStateApplier#applyClusterState} apply for the diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 0a877a9ccea4b..d193ab3c14154 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -831,7 +831,7 @@ boolean getPrefixModeVerification() { * maintains single lazy instance of {@link BlobContainer} */ protected BlobContainer blobContainer() { - assertSnapshotOrGenericThread(); + // assertSnapshotOrGenericThread(); BlobContainer blobContainer = this.blobContainer.get(); if (blobContainer == null) { @@ -3749,6 +3749,11 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In return snapshot.getIndexShardSnapshotStatus(); } + public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotInfo snapshotInfo, IndexId indexId, ShardId shardId) { + IndexShardSnapshot snapshot = loadShardSnapshot(shardContainer(indexId, shardId), snapshotInfo); + return snapshot.getIndexShardSnapshotStatus(); + } + @Override public void verify(String seed, DiscoveryNode localNode) { if (isSystemRepository == false) { @@ -3957,6 +3962,38 @@ public IndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, Snapsh } } + public IndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, SnapshotInfo snapshotInfo) { + try { + SnapshotId snapshotId = snapshotInfo.snapshotId(); + if (snapshotInfo.getPinnedTimestamp() != 0) { + return () -> IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0, 0, "1"); + } else if (snapshotInfo.isRemoteStoreIndexShallowCopyEnabled()) { + if (shardContainer.blobExists(REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) { + return REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read( + shardContainer, + snapshotId.getUUID(), + namedXContentRegistry + ); + } else { + throw new SnapshotMissingException(metadata.name(), snapshotId.getName()); + } + } else { + if (shardContainer.blobExists(INDEX_SHARD_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) { + return INDEX_SHARD_SNAPSHOT_FORMAT.read(shardContainer, snapshotId.getUUID(), namedXContentRegistry); + } else { + throw new SnapshotMissingException(metadata.name(), snapshotId.getName()); + } + } + } catch (IOException ex) { + throw new SnapshotException( + metadata.name(), + snapshotInfo.snapshotId(), + "failed to read shard snapshot file for [" + shardContainer.path() + ']', + ex + ); + } + } + /** * Loads all available snapshots in the repository using the given {@code generation} or falling back to trying to determine it from * the given list of blobs in the shard container. diff --git a/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java b/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java index 797a58f3b0d9b..e300f845e6f58 100644 --- a/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java +++ b/server/src/main/java/org/opensearch/snapshots/InternalSnapshotsInfoService.java @@ -238,14 +238,18 @@ protected void doRun() throws Exception { final Repository repository = repositories.repository(snapshotShard.snapshot.getRepository()); logger.debug("fetching snapshot shard size for {}", snapshotShard); - final long snapshotShardSize = repository.getShardSnapshotStatus( - snapshotShard.snapshot().getSnapshotId(), - snapshotShard.index(), - snapshotShard.shardId() - ).asCopy().getTotalSize(); + long snapshotShardSize; + if (snapshotShard.pinnedTimestamp > 0) { + snapshotShardSize = 0; + } else { + snapshotShardSize = repository.getShardSnapshotStatus( + snapshotShard.snapshot().getSnapshotId(), + snapshotShard.index(), + snapshotShard.shardId() + ).asCopy().getTotalSize(); + } logger.debug("snapshot shard size for {}: {} bytes", snapshotShard, snapshotShardSize); - boolean updated = false; synchronized (mutex) { removed = unknownSnapshotShards.remove(snapshotShard); @@ -354,7 +358,8 @@ private static Set listOfSnapshotShards(final ClusterState state) final SnapshotShard snapshotShard = new SnapshotShard( snapshotRecoverySource.snapshot(), snapshotRecoverySource.index(), - shardRouting.shardId() + shardRouting.shardId(), + snapshotRecoverySource.pinnedTimestamp() ); snapshotShards.add(snapshotShard); } @@ -374,10 +379,17 @@ public static class SnapshotShard { private final IndexId index; private final ShardId shardId; + private long pinnedTimestamp; + public SnapshotShard(Snapshot snapshot, IndexId index, ShardId shardId) { + this(snapshot, index, shardId, 0L); + } + + public SnapshotShard(Snapshot snapshot, IndexId index, ShardId shardId, long pinnedTimestamp) { this.snapshot = snapshot; this.index = index; this.shardId = shardId; + this.pinnedTimestamp = pinnedTimestamp; } public Snapshot snapshot() { diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 174cad50b00e2..79a70d835f773 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -428,7 +428,9 @@ public ClusterState execute(ClusterState currentState) { snapshotIndexId, isSearchableSnapshot, isRemoteStoreShallowCopy, - request.getSourceRemoteStoreRepository() + request.getSourceRemoteStoreRepository(), + request.getSourceRemoteTranslogRepository(), + snapshotInfo.getPinnedTimestamp() ); final Version minIndexCompatibilityVersion; if (isSearchableSnapshot && isSearchableSnapshotsExtendedCompatibilityEnabled()) { @@ -551,7 +553,7 @@ public ClusterState execute(ClusterState currentState) { for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) { if (isRemoteSnapshot) { IndexShardSnapshotStatus.Copy shardStatus = repository.getShardSnapshotStatus( - snapshotInfo.snapshotId(), + snapshotInfo, snapshotIndexId, new ShardId(metadata.index(index).getIndex(), shard) ).asCopy(); diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 377e4e99e9964..96794a83ef762 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2899,7 +2899,7 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { target = reinitShard(target, routing); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); target.markAsRecovering("from snapshot", new RecoveryState(routing, localNode, null)); - target.syncSegmentsFromGivenRemoteSegmentStore(false, tempRemoteSegmentDirectory, primaryTerm, commitGeneration); + target.syncSegmentsFromGivenRemoteSegmentStore(false, tempRemoteSegmentDirectory, null, false); RemoteSegmentStoreDirectory remoteStoreDirectory = ((RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) target .remoteStore() .directory()).getDelegate()).getDelegate()); diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 2426a14c0c93b..8605043ddd5b5 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -26,6 +26,7 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.TranslogReader; import org.opensearch.index.translog.transfer.FileSnapshot.CheckpointFileSnapshot; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import org.opensearch.index.translog.transfer.FileSnapshot.TranslogFileSnapshot; @@ -874,4 +875,53 @@ public void testReadMetadataForGivenTimestampException() throws IOException { assertThrows(IOException.class, () -> translogTransferManager.readMetadata(3000L)); assertNoDownloadStats(true); } + + public void testPopulateFileTrackerWithLocalStateNoReaders() { + translogTransferManager.populateFileTrackerWithLocalState(null); + assertTrue(translogTransferManager.getFileTransferTracker().allUploaded().isEmpty()); + + translogTransferManager.populateFileTrackerWithLocalState(List.of()); + assertTrue(translogTransferManager.getFileTransferTracker().allUploaded().isEmpty()); + } + + public void testPopulateFileTrackerWithLocalState() { + TranslogReader reader1 = mock(TranslogReader.class); + when(reader1.getGeneration()).thenReturn(12L); + TranslogReader reader2 = mock(TranslogReader.class); + when(reader2.getGeneration()).thenReturn(23L); + TranslogReader reader3 = mock(TranslogReader.class); + when(reader3.getGeneration()).thenReturn(34L); + TranslogReader reader4 = mock(TranslogReader.class); + when(reader4.getGeneration()).thenReturn(45L); + + translogTransferManager.populateFileTrackerWithLocalState(List.of(reader1, reader2, reader3, reader4)); + assertEquals( + Set.of("translog-12.tlog", "translog-23.tlog", "translog-34.tlog", "translog-45.tlog"), + translogTransferManager.getFileTransferTracker().allUploaded() + ); + } + + public void testPopulateFileTrackerWithLocalStateNoCkpAsMetadata() { + TranslogTransferManager translogTransferManager = new TranslogTransferManager( + shardId, + transferService, + remoteBaseTransferPath.add(TRANSLOG.getName()), + remoteBaseTransferPath.add(METADATA.getName()), + tracker, + remoteTranslogTransferTracker, + DefaultRemoteStoreSettings.INSTANCE, + true + ); + + TranslogReader reader1 = mock(TranslogReader.class); + when(reader1.getGeneration()).thenReturn(12L); + TranslogReader reader2 = mock(TranslogReader.class); + when(reader2.getGeneration()).thenReturn(23L); + + translogTransferManager.populateFileTrackerWithLocalState(List.of(reader1, reader2)); + assertEquals( + Set.of("translog-12.tlog", "translog-12.ckp", "translog-23.tlog", "translog-23.ckp"), + translogTransferManager.getFileTransferTracker().allUploaded() + ); + } } From 451606535752a73be80d5203ae417e7d57fc5cef Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Tue, 3 Sep 2024 20:11:02 +0530 Subject: [PATCH 024/357] Limiting node attribute serialisation in DiscoveryNode (#15341) Signed-off-by: RS146BIJAY --- .../core/common/io/stream/StreamOutput.java | 6 ++++- .../ClusterAllocationExplanation.java | 2 +- .../support/nodes/BaseNodeResponse.java | 2 +- .../org/opensearch/cluster/ClusterState.java | 14 ++++++++-- .../opensearch/cluster/coordination/Join.java | 4 +-- .../cluster/coordination/JoinRequest.java | 2 +- .../coordination/StartJoinRequest.java | 2 +- .../cluster/node/DiscoveryNode.java | 27 ++++++++++++++++--- .../cluster/node/DiscoveryNodes.java | 10 ++++++- .../AbstractAllocationDecision.java | 2 +- .../allocation/NodeAllocationResult.java | 2 +- .../remote/model/RemoteDiscoveryNodes.java | 7 ++++- .../recovery/PeerRecoveryTargetService.java | 1 + .../recovery/StartRecoveryRequest.java | 4 +-- .../ChecksumWritableBlobStoreFormat.java | 7 ++++- .../transport/TransportService.java | 2 +- ...oteClusterStateAttributesManagerTests.java | 7 ++++- .../RemoteClusterStateServiceTests.java | 7 ++++- .../model/RemoteDiscoveryNodesTests.java | 2 +- .../ChecksumWritableBlobStoreFormatTests.java | 8 +++++- 20 files changed, 93 insertions(+), 25 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java index 867fc99cc6a56..cac8ddc8f94e3 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java @@ -969,9 +969,13 @@ public void writeOptionalArray(@Nullable T[] array) throws } public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException { + writeOptionalWriteable((out, writable) -> writable.writeTo(out), writeable); + } + + public void writeOptionalWriteable(final Writer writer, @Nullable T writeable) throws IOException { if (writeable != null) { writeBoolean(true); - writeable.writeTo(this); + writer.write(this, writeable); } else { writeBoolean(false); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java index 3c8f07613561d..70a223d60069a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java @@ -95,7 +95,7 @@ public ClusterAllocationExplanation(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { shardRouting.writeTo(out); - out.writeOptionalWriteable(currentNode); + out.writeOptionalWriteable((stream, node) -> node.writeToWithAttribute(stream), currentNode); out.writeOptionalWriteable(relocationTargetNode); out.writeOptionalWriteable(clusterInfo); shardAllocationDecision.writeTo(out); diff --git a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java index 8a4e12567b515..7e10b583ef21a 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/BaseNodeResponse.java @@ -67,6 +67,6 @@ public DiscoveryNode getNode() { @Override public void writeTo(StreamOutput out) throws IOException { - node.writeTo(out); + node.writeToWithAttribute(out); } } diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 054233305726d..1e4fd2dfffe0f 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -781,7 +781,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(stateUUID); metadata.writeTo(out); routingTable.writeTo(out); - nodes.writeTo(out); + nodes.writeToWithAttribute(out); blocks.writeTo(out); // filter out custom states not supported by the other node int numberOfCustoms = 0; @@ -887,13 +887,23 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(toUuid); out.writeLong(toVersion); routingTable.writeTo(out); - nodes.writeTo(out); + nodesWriteToWithAttributes(nodes, out); metadata.writeTo(out); blocks.writeTo(out); customs.writeTo(out); out.writeVInt(minimumClusterManagerNodesOnPublishingClusterManager); } + private void nodesWriteToWithAttributes(Diff nodes, StreamOutput out) throws IOException { + DiscoveryNodes part = nodes.apply(null); + if (part != null) { + out.writeBoolean(true); + part.writeToWithAttribute(out); + } else { + out.writeBoolean(false); + } + } + @Override public ClusterState apply(ClusterState state) { Builder builder = new Builder(clusterName); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Join.java b/server/src/main/java/org/opensearch/cluster/coordination/Join.java index 58fa85992ebc8..ce1a234998690 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Join.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Join.java @@ -78,8 +78,8 @@ public Join(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - sourceNode.writeTo(out); - targetNode.writeTo(out); + sourceNode.writeToWithAttribute(out); + targetNode.writeToWithAttribute(out); out.writeLong(term); out.writeLong(lastAcceptedTerm); out.writeLong(lastAcceptedVersion); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java index 04f87d16ee400..1447838a41502 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinRequest.java @@ -84,7 +84,7 @@ public JoinRequest(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - sourceNode.writeTo(out); + sourceNode.writeToWithAttribute(out); out.writeLong(minimumTerm); out.writeOptionalWriteable(optionalJoin.orElse(null)); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java index de58eb721b28f..287418aaf378e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/StartJoinRequest.java @@ -64,7 +64,7 @@ public StartJoinRequest(StreamInput input) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - sourceNode.writeTo(out); + sourceNode.writeToWithAttribute(out); out.writeLong(term); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 50086d458fab2..8ac978a72b21b 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -330,6 +330,7 @@ public DiscoveryNode(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { this.attributes.put(in.readString(), in.readString()); } + int rolesSize = in.readVInt(); final Set roles = new HashSet<>(rolesSize); for (int i = 0; i < rolesSize; i++) { @@ -359,13 +360,31 @@ public DiscoveryNode(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + writeToUtil(out, false); + } else { + writeToUtil(out, true); + } + + } + + public void writeToWithAttribute(StreamOutput out) throws IOException { + writeToUtil(out, true); + } + + public void writeToUtil(StreamOutput out, boolean includeAllAttributes) throws IOException { writeNodeDetails(out); - out.writeVInt(attributes.size()); - for (Map.Entry entry : attributes.entrySet()) { - out.writeString(entry.getKey()); - out.writeString(entry.getValue()); + if (includeAllAttributes) { + out.writeVInt(attributes.size()); + for (Map.Entry entry : attributes.entrySet()) { + out.writeString(entry.getKey()); + out.writeString(entry.getValue()); + } + } else { + out.writeVInt(0); } + writeRolesAndVersion(out); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index ba81012e386d7..52d830aafda38 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -690,10 +690,18 @@ public String shortSummary() { @Override public void writeTo(StreamOutput out) throws IOException { + writeToUtil((output, value) -> value.writeTo(output), out); + } + + public void writeToWithAttribute(StreamOutput out) throws IOException { + writeToUtil((output, value) -> value.writeToWithAttribute(output), out); + } + + public void writeToUtil(final Writer writer, StreamOutput out) throws IOException { writeClusterManager(out); out.writeVInt(nodes.size()); for (DiscoveryNode node : this) { - node.writeTo(out); + writer.write(out, node); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java index 59a39b358cb70..614e9f49c8726 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AbstractAllocationDecision.java @@ -107,7 +107,7 @@ public List getNodeDecisions() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalWriteable(targetNode); + out.writeOptionalWriteable((stream, node) -> node.writeToWithAttribute(stream), targetNode); if (nodeDecisions != null) { out.writeBoolean(true); out.writeList(nodeDecisions); diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java index 4163a5fd4c16f..6b805ca91fa58 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java @@ -104,7 +104,7 @@ public NodeAllocationResult(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - node.writeTo(out); + node.writeToWithAttribute(out); out.writeOptionalWriteable(shardStoreInfo); out.writeOptionalWriteable(canAllocateDecision); nodeDecision.writeTo(out); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 446207a767009..829036c6d122b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -88,7 +88,12 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor()).streamInput(); + return DISCOVERY_NODES_FORMAT.serialize( + (out, discoveryNode) -> discoveryNode.writeToWithAttribute(out), + discoveryNodes, + generateBlobFileName(), + getCompressor() + ).streamInput(); } @Override diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index 6279a8ec3646c..d746aaa2a0783 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -294,6 +294,7 @@ private void doRecovery(final long recoveryId, final StartRecoveryRequest preExi logger.debug("{} reestablishing recovery from {}", startRequest.shardId(), startRequest.sourceNode()); } } + transportService.sendRequest( startRequest.sourceNode(), actionName, diff --git a/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java b/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java index 1df0d3861f686..7309712314acd 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java +++ b/server/src/main/java/org/opensearch/indices/recovery/StartRecoveryRequest.java @@ -144,8 +144,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(recoveryId); shardId.writeTo(out); out.writeString(targetAllocationId); - sourceNode.writeTo(out); - targetNode.writeTo(out); + sourceNode.writeToWithAttribute(out); + targetNode.writeToWithAttribute(out); metadataSnapshot.writeTo(out); out.writeBoolean(primaryRelocation); out.writeLong(startingSeqNo); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java index 0add86ab88a16..88672995f4fd6 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java @@ -28,6 +28,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable.Writer; import org.opensearch.core.compress.Compressor; import org.opensearch.core.compress.CompressorRegistry; import org.opensearch.gateway.CorruptStateException; @@ -56,6 +57,10 @@ public ChecksumWritableBlobStoreFormat(String codec, CheckedFunction unSerializedObj.writeTo(out), obj, blobName, compressor); + } + + public BytesReference serialize(final Writer writer, T obj, final String blobName, final Compressor compressor) throws IOException { try (BytesStreamOutput outputStream = new BytesStreamOutput()) { try ( OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput( @@ -76,7 +81,7 @@ public void close() throws IOException { }; StreamOutput stream = new OutputStreamStreamOutput(compressor.threadLocalOutputStream(indexOutputOutputStream));) { // TODO The stream version should be configurable stream.setVersion(Version.CURRENT); - obj.writeTo(stream); + writer.write(stream, obj); } CodecUtil.writeFooter(indexOutput); } diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index d08b28730d417..fff6d82b23c7e 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -752,7 +752,7 @@ public HandshakeResponse(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalWriteable(discoveryNode); + out.writeOptionalWriteable((stream, node) -> node.writeToWithAttribute(stream), discoveryNode); clusterName.writeTo(out); out.writeVersion(version); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java index 4ef459e6657a1..67b1528466a9e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManagerTests.java @@ -140,7 +140,12 @@ public void testGetAsyncReadRunnable_DiscoveryNodes() throws IOException, Interr DiscoveryNodes discoveryNodes = getDiscoveryNodes(); String fileName = randomAlphaOfLength(10); when(blobStoreTransferService.downloadBlob(anyIterable(), anyString())).thenReturn( - DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, fileName, compressor).streamInput() + DISCOVERY_NODES_FORMAT.serialize( + (out, discoveryNode) -> discoveryNode.writeToWithAttribute(out), + discoveryNodes, + fileName, + compressor + ).streamInput() ); RemoteDiscoveryNodes remoteObjForDownload = new RemoteDiscoveryNodes(fileName, "cluster-uuid", compressor); CountDownLatch latch = new CountDownLatch(1); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 8dd35e59c43c9..ccadbfb3fbebc 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1220,7 +1220,12 @@ public void testGetClusterStateUsingDiff() throws IOException { diffManifestBuilder.discoveryNodesUpdated(true); manifestBuilder.discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, DISCOVERY_NODES_FILENAME)); when(blobContainer.readBlob(DISCOVERY_NODES_FILENAME)).thenAnswer(invocationOnMock -> { - BytesReference bytes = DISCOVERY_NODES_FORMAT.serialize(nodesBuilder.build(), DISCOVERY_NODES_FILENAME, compressor); + BytesReference bytes = DISCOVERY_NODES_FORMAT.serialize( + (out, nodes) -> nodes.writeToWithAttribute(out), + nodesBuilder.build(), + DISCOVERY_NODES_FILENAME, + compressor + ); return new ByteArrayInputStream(bytes.streamInput().readAllBytes()); }); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java index f1bced2bdf855..1b988ee1f37ec 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java @@ -143,7 +143,7 @@ public void testSerDe() throws IOException { public void testExceptionDuringSerialization() throws IOException { DiscoveryNodes nodes = mock(DiscoveryNodes.class); RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); - doThrow(new IOException("mock-exception")).when(nodes).writeTo(any()); + doThrow(new IOException("mock-exception")).when(nodes).writeToWithAttribute(any()); IOException iea = assertThrows(IOException.class, remoteObjectForUpload::serialize); } diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java index 536df880b2597..c4e53c1eea138 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java @@ -35,7 +35,12 @@ public class ChecksumWritableBlobStoreFormatTests extends OpenSearchTestCase { public void testSerDe() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - BytesReference bytesReference = clusterBlocksFormat.serialize(indexMetadata, TEST_BLOB_FILE_NAME, CompressorRegistry.none()); + BytesReference bytesReference = clusterBlocksFormat.serialize( + (out, metadata) -> metadata.writeTo(out), + indexMetadata, + TEST_BLOB_FILE_NAME, + CompressorRegistry.none() + ); IndexMetadata readIndexMetadata = clusterBlocksFormat.deserialize(TEST_BLOB_FILE_NAME, bytesReference); assertThat(readIndexMetadata, is(indexMetadata)); } @@ -43,6 +48,7 @@ public void testSerDe() throws IOException { public void testSerDeForCompressed() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); BytesReference bytesReference = clusterBlocksFormat.serialize( + (out, metadata) -> metadata.writeTo(out), indexMetadata, TEST_BLOB_FILE_NAME, CompressorRegistry.getCompressor(DeflateCompressor.NAME) From 12f6493ec4f489bf1932b2ce757c28ebafbb0ca8 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 3 Sep 2024 21:30:39 +0530 Subject: [PATCH 025/357] interfaces fixes for star tree search (#15603) Signed-off-by: Sarthak Aggarwal --- .../fileformats/meta/StarTreeMetadata.java | 2 ++ .../node/FixedLengthStarTreeNode.java | 1 - .../startree/index/StarTreeValues.java | 19 ++++++++--- .../datacube/startree/StarTreeTestUtils.java | 8 ++--- .../builder/AbstractStarTreeBuilderTests.java | 33 +++++++++++++++---- .../node/FixedLengthStarTreeNodeTests.java | 2 +- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 2e20c9b9e0737..e7bb32282ece7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.IndexInput; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -30,6 +31,7 @@ * * @opensearch.experimental */ +@ExperimentalApi public class StarTreeMetadata extends CompositeIndexMetadata { private static final Logger logger = LogManager.getLogger(StarTreeMetadata.class); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 72c08d96d26ff..0ae704afa53b9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -201,7 +201,6 @@ public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOExce StarTreeNode resultStarTreeNode = null; if (null != dimensionValue) { resultStarTreeNode = binarySearchChild(dimensionValue); - assert null != resultStarTreeNode; } return resultStarTreeNode; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 8181a69f49bd4..7d5f5ba02b9f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -9,7 +9,6 @@ package org.opensearch.index.compositeindex.datacube.startree.index; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; @@ -74,6 +73,11 @@ public class StarTreeValues implements CompositeIndexValues { */ private final Map attributes; + /** + * A metadata for the star-tree + */ + private final StarTreeMetadata starTreeMetadata; + /** * Constructs a new StarTreeValues object with the provided parameters. * Used for testing. @@ -89,13 +93,15 @@ public StarTreeValues( StarTreeNode root, Map> dimensionDocValuesIteratorMap, Map> metricDocValuesIteratorMap, - Map attributes + Map attributes, + StarTreeMetadata compositeIndexMetadata ) { this.starTreeField = starTreeField; this.root = root; this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; this.attributes = attributes; + this.starTreeMetadata = compositeIndexMetadata; } /** @@ -114,7 +120,7 @@ public StarTreeValues( SegmentReadState readState ) throws IOException { - StarTreeMetadata starTreeMetadata = (StarTreeMetadata) compositeIndexMetadata; + starTreeMetadata = (StarTreeMetadata) compositeIndexMetadata; // build skip star node dimensions Set skipStarNodeCreationInDims = starTreeMetadata.getSkipStarNodeCreationInDims(); @@ -244,7 +250,7 @@ public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { return dimensionDocValuesIteratorMap.get(dimension).get(); } - return DocValues.emptySortedNumeric(); + throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); } /** @@ -259,7 +265,10 @@ public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricNam return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); } - return DocValues.emptySortedNumeric(); + throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); } + public int getStarTreeDocumentCount() { + return starTreeMetadata.getStarTreeDocCount(); + } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 53c9e05ccb490..7b5672529dc8d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -37,7 +37,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class StarTreeTestUtils { @@ -208,11 +208,7 @@ public static void validateFileFormats( if (child.getStarTreeNodeType() != StarTreeNodeType.NULL.getValue()) { assertNotNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); } else { - StarTreeNode finalStarTreeNode = starTreeNode; - assertThrows( - AssertionError.class, - () -> finalStarTreeNode.getChildForDimensionValue(child.getDimensionValue()) - ); + assertNull(starTreeNode.getChildForDimensionValue(child.getDimensionValue())); } assertStarTreeNode(child, resultChildNode); assertNotEquals(child.getStarTreeNodeType(), StarTreeNodeType.STAR.getValue()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 79d1b0e4b9785..6e23e5eacaecd 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -1894,6 +1894,7 @@ private void validateStarTreeFileFormats( IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); + assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), starTreeValues.getStarTreeDocumentCount()); List starTreeNumericTypes = new ArrayList<>(); builder.metricAggregatorInfos.forEach( metricAggregatorInfo -> starTreeNumericTypes.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) @@ -2527,7 +2528,8 @@ private StarTreeValues getStarTreeValues( null, dimDocIdSetIterators, metricDocIdSetIterators, - Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number) + Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number), + null ); return starTreeValues; } @@ -3678,7 +3680,14 @@ private StarTreeValues getStarTreeValues( ); // metricDocIdSetIterators.put("field2", () -> m1sndv); // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); - StarTreeValues starTreeValues = new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, getAttributes(500)); + StarTreeValues starTreeValues = new StarTreeValues( + sf, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(500), + null + ); return starTreeValues; } @@ -4088,14 +4097,20 @@ public void testMergeFlow() throws IOException { () -> d4sndv ); - Map> metricDocIdSetIterators = Map.of("field2", () -> m1sndv, "_doc_count", () -> m2sndv); + Map> metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> m1sndv, + "sf__doc_count_doc_count_metric", + () -> m2sndv + ); StarTreeValues starTreeValues = new StarTreeValues( compositeField, null, dimDocIdSetIterators, metricDocIdSetIterators, - getAttributes(1000) + getAttributes(1000), + null ); SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); @@ -4115,13 +4130,19 @@ public void testMergeFlow() throws IOException { () -> f2d4sndv ); - Map> f2metricDocIdSetIterators = Map.of("field2", () -> f2m1sndv, "_doc_count", () -> f2m2sndv); + Map> f2metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> f2m1sndv, + "sf__doc_count_doc_count_metric", + () -> f2m2sndv + ); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, null, f2dimDocIdSetIterators, f2metricDocIdSetIterators, - getAttributes(1000) + getAttributes(1000), + null ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index 9657f867422a0..08815d5ef55f5 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -191,7 +191,7 @@ public void testGetChildForNullNode() throws IOException { public void testGetChildForInvalidDimensionValue() throws IOException { long invalidDimensionValue = Long.MAX_VALUE; - assertThrows(AssertionError.class, () -> starTreeNode.getChildForDimensionValue(invalidDimensionValue)); + assertNull(starTreeNode.getChildForDimensionValue(invalidDimensionValue)); } public void testOnlyRootNodePresent() throws IOException { From c62626d25430bea94123d6176caef56ae94deb3f Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Tue, 3 Sep 2024 22:09:17 +0530 Subject: [PATCH 026/357] SnapshotInfo pinned timestamp version change to V_2_17_0 (#15616) Signed-off-by: Anshu Agarwal Co-authored-by: Anshu Agarwal --- .../src/main/java/org/opensearch/snapshots/SnapshotInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java index bc33928acf3da..08433dc6f2e0b 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java @@ -473,7 +473,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_9_0)) { remoteStoreIndexShallowCopy = in.readOptionalBoolean(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { pinnedTimestamp = in.readVLong(); } } @@ -940,7 +940,7 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeOptionalBoolean(remoteStoreIndexShallowCopy); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeVLong(pinnedTimestamp); } } From deeb2ded71f05652900ca2915623fd3d0c81c97f Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 3 Sep 2024 23:26:35 +0530 Subject: [PATCH 027/357] Abstract out RemoteFsTranslogWithPinnedTimestamps (#15579) * Abstract out RemoteFsTranslog with pinned timestamps Signed-off-by: Sachin Kale --- ...emoteBlobStoreInternalTranslogFactory.java | 42 +- .../RemoteFsTimestampAwareTranslog.java | 377 ++++++++++++++++++ .../index/translog/RemoteFsTranslog.java | 309 ++------------ ... RemoteFsTimestampAwareTranslogTests.java} | 75 +++- .../index/translog/RemoteFsTranslogTests.java | 38 +- 5 files changed, 531 insertions(+), 310 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java rename server/src/test/java/org/opensearch/index/translog/{RemoteFsTranslogWithPinnedTimestampTests.java => RemoteFsTimestampAwareTranslogTests.java} (93%) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java index 4599aa32325c1..5dc2ad076d21c 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java @@ -69,19 +69,35 @@ public Translog newTranslog( assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); - return new RemoteFsTranslog( - config, - translogUUID, - deletionPolicy, - globalCheckpointSupplier, - primaryTermSupplier, - persistedSequenceNumberConsumer, - blobStoreRepository, - threadPool, - startedPrimarySupplier, - remoteTranslogTransferTracker, - remoteStoreSettings - ); + if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { + return new RemoteFsTimestampAwareTranslog( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + blobStoreRepository, + threadPool, + startedPrimarySupplier, + remoteTranslogTransferTracker, + remoteStoreSettings + ); + } else { + return new RemoteFsTranslog( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + blobStoreRepository, + threadPool, + startedPrimarySupplier, + remoteTranslogTransferTracker, + remoteStoreSettings + ); + } } public Repository getRepository() { diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java new file mode 100644 index 0000000000000..0b134b3bddbec --- /dev/null +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -0,0 +1,377 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.logging.Loggers; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.remote.RemoteTranslogTransferTracker; +import org.opensearch.index.translog.transfer.TranslogTransferManager; +import org.opensearch.index.translog.transfer.TranslogTransferMetadata; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.BooleanSupplier; +import java.util.function.LongConsumer; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; + +/** + * A Translog implementation which syncs local FS with a remote store + * The current impl uploads translog , ckp and metadata to remote store + * for every sync, post syncing to disk. Post that, a new generation is + * created. This implementation is also aware of pinned timestamp and makes + * sure data against pinned timestamp is retained. + * + * @opensearch.internal + */ +public class RemoteFsTimestampAwareTranslog extends RemoteFsTranslog { + + private final Logger logger; + private final Map metadataFilePinnedTimestampMap; + // For metadata files, with no min generation in the name, we cache generation data to avoid multiple reads. + private final Map> oldFormatMetadataFileGenerationMap; + + public RemoteFsTimestampAwareTranslog( + TranslogConfig config, + String translogUUID, + TranslogDeletionPolicy deletionPolicy, + LongSupplier globalCheckpointSupplier, + LongSupplier primaryTermSupplier, + LongConsumer persistedSequenceNumberConsumer, + BlobStoreRepository blobStoreRepository, + ThreadPool threadPool, + BooleanSupplier startedPrimarySupplier, + RemoteTranslogTransferTracker remoteTranslogTransferTracker, + RemoteStoreSettings remoteStoreSettings + ) throws IOException { + super( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + blobStoreRepository, + threadPool, + startedPrimarySupplier, + remoteTranslogTransferTracker, + remoteStoreSettings + ); + logger = Loggers.getLogger(getClass(), shardId); + this.metadataFilePinnedTimestampMap = new HashMap<>(); + this.oldFormatMetadataFileGenerationMap = new HashMap<>(); + } + + @Override + protected void onDelete() { + ClusterService.assertClusterOrClusterManagerStateThread(); + // clean up all remote translog files + try { + trimUnreferencedReaders(true, false); + } catch (IOException e) { + logger.error("Exception while deleting translog files from remote store", e); + } + } + + @Override + public void trimUnreferencedReaders() throws IOException { + trimUnreferencedReaders(false, true); + } + + // Visible for testing + protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException { + if (trimLocal) { + // clean up local translog files and updates readers + super.trimUnreferencedReaders(); + } + + // Update file tracker to reflect local translog state + Optional minLiveGeneration = readers.stream().map(BaseTranslogReader::getGeneration).min(Long::compareTo); + if (minLiveGeneration.isPresent()) { + List staleFilesInTracker = new ArrayList<>(); + for (String file : fileTransferTracker.allUploaded()) { + if (file.endsWith(TRANSLOG_FILE_SUFFIX)) { + long generation = Translog.parseIdFromFileName(file); + if (generation < minLiveGeneration.get()) { + staleFilesInTracker.add(file); + staleFilesInTracker.add(Translog.getCommitCheckpointFileName(generation)); + } + } + fileTransferTracker.delete(staleFilesInTracker); + } + } + + // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote + // store. + if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) { + return; + } + + // This is to fail fast and avoid listing md files un-necessarily. + if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { + logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + return; + } + + // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time. + // Remote generations involves 2 async operations - 1) Delete translog generation files 2) Delete metadata files + // We try to acquire 2 permits and if we can not, we return from here itself. + if (remoteGenerationDeletionPermits.tryAcquire(REMOTE_DELETION_PERMITS) == false) { + return; + } + + ActionListener> listMetadataFilesListener = new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + + try { + if (metadataFiles.size() <= 1) { + logger.debug("No stale translog metadata files found"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + // Check last fetch status of pinned timestamps. If stale, return. + if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { + logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles); + + // If index is not deleted, make sure to keep latest metadata file + if (indexDeleted == false) { + metadataFilesToBeDeleted.remove(metadataFiles.get(0)); + } + + if (metadataFilesToBeDeleted.isEmpty()) { + logger.debug("No metadata files to delete"); + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + return; + } + + logger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + // For all the files that we are keeping, fetch min and max generations + List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + + logger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + Set generationsToBeDeleted = getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + indexDeleted + ); + + logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted); + if (generationsToBeDeleted.isEmpty() == false) { + // Delete stale generations + translogTransferManager.deleteGenerationAsync( + primaryTermSupplier.getAsLong(), + generationsToBeDeleted, + remoteGenerationDeletionPermits::release + ); + + // Delete stale metadata files + translogTransferManager.deleteMetadataFilesAsync( + metadataFilesToBeDeleted, + remoteGenerationDeletionPermits::release + ); + + // Update cache to keep only those metadata files that are not getting deleted + oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); + + // Delete stale primary terms + deleteStaleRemotePrimaryTerms(metadataFiles); + } else { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + } + } catch (Exception e) { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + } + } + + @Override + public void onFailure(Exception e) { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + logger.error("Exception while listing translog metadata files", e); + } + }; + translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + } + + // Visible for testing + protected Set getGenerationsToBeDeleted( + List metadataFilesNotToBeDeleted, + List metadataFilesToBeDeleted, + boolean indexDeleted + ) throws IOException { + long maxGenerationToBeDeleted = Long.MAX_VALUE; + + if (indexDeleted == false) { + maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); + } + + Set generationsFromMetadataFilesToBeDeleted = new HashSet<>(); + for (String mdFile : metadataFilesToBeDeleted) { + Tuple minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager); + generationsFromMetadataFilesToBeDeleted.addAll( + LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList()) + ); + } + + Map> metadataFileNotToBeDeletedGenerationMap = getGenerationForMetadataFiles(metadataFilesNotToBeDeleted); + TreeSet> pinnedGenerations = getOrderedPinnedMetadataGenerations(metadataFileNotToBeDeletedGenerationMap); + Set generationsToBeDeleted = new HashSet<>(); + for (long generation : generationsFromMetadataFilesToBeDeleted) { + // Check if the generation is not referred by metadata file matching pinned timestamps + if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) { + generationsToBeDeleted.add(generation); + } + } + return generationsToBeDeleted; + } + + // Visible for testing + protected List getMetadataFilesToBeDeleted(List metadataFiles) { + Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); + + // Keep files since last successful run of scheduler + List metadataFilesToBeDeleted = RemoteStoreUtils.filterOutMetadataFilesBasedOnAge( + metadataFiles, + file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), + pinnedTimestampsState.v1() + ); + + logger.trace( + "metadataFiles.size = {}, metadataFilesToBeDeleted based on age based filtering = {}", + metadataFiles.size(), + metadataFilesToBeDeleted.size() + ); + + // Get md files matching pinned timestamps + Set implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles( + metadataFilesToBeDeleted, + pinnedTimestampsState.v2(), + metadataFilePinnedTimestampMap, + file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), + TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen + ); + + // Filter out metadata files matching pinned timestamps + metadataFilesToBeDeleted.removeAll(implicitLockedFiles); + + logger.trace( + "implicitLockedFiles.size = {}, metadataFilesToBeDeleted based on pinned timestamp filtering = {}", + implicitLockedFiles.size(), + metadataFilesToBeDeleted.size() + ); + + return metadataFilesToBeDeleted; + } + + // Visible for testing + protected boolean isGenerationPinned(long generation, TreeSet> pinnedGenerations) { + Tuple ceilingGenerationRange = pinnedGenerations.ceiling(new Tuple<>(generation, generation)); + if (ceilingGenerationRange != null && generation >= ceilingGenerationRange.v1() && generation <= ceilingGenerationRange.v2()) { + return true; + } + Tuple floorGenerationRange = pinnedGenerations.floor(new Tuple<>(generation, generation)); + if (floorGenerationRange != null && generation >= floorGenerationRange.v1() && generation <= floorGenerationRange.v2()) { + return true; + } + return false; + } + + private TreeSet> getOrderedPinnedMetadataGenerations(Map> metadataFileGenerationMap) { + TreeSet> pinnedGenerations = new TreeSet<>((o1, o2) -> { + if (Objects.equals(o1.v1(), o2.v1()) == false) { + return o1.v1().compareTo(o2.v1()); + } else { + return o1.v2().compareTo(o2.v2()); + } + }); + pinnedGenerations.addAll(metadataFileGenerationMap.values()); + return pinnedGenerations; + } + + // Visible for testing + protected Map> getGenerationForMetadataFiles(List metadataFiles) throws IOException { + Map> metadataFileGenerationMap = new HashMap<>(); + for (String metadataFile : metadataFiles) { + metadataFileGenerationMap.put(metadataFile, getMinMaxTranslogGenerationFromMetadataFile(metadataFile, translogTransferManager)); + } + return metadataFileGenerationMap; + } + + // Visible for testing + protected Tuple getMinMaxTranslogGenerationFromMetadataFile( + String metadataFile, + TranslogTransferManager translogTransferManager + ) throws IOException { + Tuple minMaxGenerationFromFileName = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(metadataFile); + if (minMaxGenerationFromFileName != null) { + return minMaxGenerationFromFileName; + } else { + if (oldFormatMetadataFileGenerationMap.containsKey(metadataFile)) { + return oldFormatMetadataFileGenerationMap.get(metadataFile); + } else { + TranslogTransferMetadata metadata = translogTransferManager.readMetadata(metadataFile); + Tuple minMaxGenTuple = new Tuple<>(metadata.getMinTranslogGeneration(), metadata.getGeneration()); + oldFormatMetadataFileGenerationMap.put(metadataFile, minMaxGenTuple); + return minMaxGenTuple; + } + } + } + + /** + * This method must be called only after there are valid generations to delete in trimUnreferencedReaders as it ensures + * implicitly that minimum primary term in latest translog metadata in remote store is the current primary term. + *
+ * This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. + */ + private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there + // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part + // of older primary term. + if (olderPrimaryCleaned.trySet(Boolean.TRUE)) { + if (metadataFiles.isEmpty()) { + logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); + return; + } + Optional minPrimaryTerm = metadataFiles.stream() + .map(file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[1])) + .min(Long::compareTo); + // First we delete all stale primary terms folders from remote store + long minimumReferencedPrimaryTerm = minPrimaryTerm.get() - 1; + translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 16ebac0a44139..812852d107682 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -11,19 +11,15 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; -import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.logging.Loggers; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.index.remote.RemoteStorePathStrategy; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -34,7 +30,6 @@ import org.opensearch.index.translog.transfer.TranslogTransferMetadata; import org.opensearch.index.translog.transfer.listener.TranslogTransferListener; import org.opensearch.indices.RemoteStoreSettings; -import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -44,16 +39,11 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -61,13 +51,10 @@ import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; import java.util.function.LongSupplier; -import java.util.stream.Collectors; -import java.util.stream.LongStream; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; -import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; /** * A Translog implementation which syncs local FS with a remote store @@ -80,36 +67,31 @@ public class RemoteFsTranslog extends Translog { private final Logger logger; - private final TranslogTransferManager translogTransferManager; - // This tracker keeps track of local tranlog files that are uploaded to remote store. - // Once tlog files are deleted from local, we remove them from tracker even if the files still exist in remote translog. - private final FileTransferTracker fileTransferTracker; - private final BooleanSupplier startedPrimarySupplier; + protected final TranslogTransferManager translogTransferManager; + protected final FileTransferTracker fileTransferTracker; + protected final BooleanSupplier startedPrimarySupplier; private final RemoteTranslogTransferTracker remoteTranslogTransferTracker; private volatile long maxRemoteTranslogGenerationUploaded; private volatile long minSeqNoToKeep; // min generation referred by last uploaded translog - private volatile long minRemoteGenReferenced; + protected volatile long minRemoteGenReferenced; // clean up translog folder uploaded by previous primaries once - private final SetOnce olderPrimaryCleaned = new SetOnce<>(); + protected final SetOnce olderPrimaryCleaned = new SetOnce<>(); - private static final int REMOTE_DELETION_PERMITS = 2; + protected static final int REMOTE_DELETION_PERMITS = 2; private static final int DOWNLOAD_RETRIES = 2; // Semaphore used to allow only single remote generation to happen at a time - private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS); + protected final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS); // These permits exist to allow any inflight background triggered upload. private static final int SYNC_PERMIT = 1; private final Semaphore syncPermit = new Semaphore(SYNC_PERMIT); - private final AtomicBoolean pauseSync = new AtomicBoolean(false); + protected final AtomicBoolean pauseSync = new AtomicBoolean(false); private final boolean isTranslogMetadataEnabled; - private final Map metadataFilePinnedTimestampMap; - // For metadata files, with no min generation in the name, we cache generation data to avoid multiple reads. - private final Map> oldFormatMetadataFileGenerationMap; public RemoteFsTranslog( TranslogConfig config, @@ -130,8 +112,6 @@ public RemoteFsTranslog( this.remoteTranslogTransferTracker = remoteTranslogTransferTracker; fileTransferTracker = new FileTransferTracker(shardId, remoteTranslogTransferTracker); isTranslogMetadataEnabled = indexSettings().isTranslogMetadataEnabled(); - this.metadataFilePinnedTimestampMap = new HashMap<>(); - this.oldFormatMetadataFileGenerationMap = new HashMap<>(); this.translogTransferManager = buildTranslogTransferManager( blobStoreRepository, threadPool, @@ -569,31 +549,8 @@ protected Releasable drainSync() { @Override public void trimUnreferencedReaders() throws IOException { - trimUnreferencedReaders(false, true); - } - - // Visible for testing - protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException { - if (trimLocal) { - // clean up local translog files and updates readers - super.trimUnreferencedReaders(); - } - - // Update file tracker to reflect local translog state - Optional minLiveGeneration = readers.stream().map(BaseTranslogReader::getGeneration).min(Long::compareTo); - if (minLiveGeneration.isPresent()) { - List staleFilesInTracker = new ArrayList<>(); - for (String file : fileTransferTracker.allUploaded()) { - if (file.endsWith(TRANSLOG_FILE_SUFFIX)) { - long generation = Translog.parseIdFromFileName(file); - if (generation < minLiveGeneration.get()) { - staleFilesInTracker.add(file); - staleFilesInTracker.add(Translog.getCommitCheckpointFileName(generation)); - } - } - fileTransferTracker.delete(staleFilesInTracker); - } - } + // clean up local translog files and updates readers + super.trimUnreferencedReaders(); // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. @@ -601,12 +558,6 @@ protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) return; } - // This is to fail fast and avoid listing md files un-necessarily. - if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); - return; - } - // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time. // Remote generations involves 2 async operations - 1) Delete translog generation files 2) Delete metadata files // We try to acquire 2 permits and if we can not, we return from here itself. @@ -614,209 +565,34 @@ protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) return; } - ActionListener> listMetadataFilesListener = new ActionListener<>() { - @Override - public void onResponse(List blobMetadata) { - List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); - - try { - if (metadataFiles.size() <= 1) { - logger.debug("No stale translog metadata files found"); - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - return; - } - - // Check last fetch status of pinned timestamps. If stale, return. - if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - return; - } - - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles); - - // If index is not deleted, make sure to keep latest metadata file - if (indexDeleted == false) { - metadataFilesToBeDeleted.remove(metadataFiles.get(0)); - } - - if (metadataFilesToBeDeleted.isEmpty()) { - logger.debug("No metadata files to delete"); - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - return; - } - - logger.debug("metadataFilesToBeDeleted = {}", metadataFilesToBeDeleted); - // For all the files that we are keeping, fetch min and max generations - List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); - metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); - - logger.debug("metadataFilesNotToBeDeleted = {}", metadataFilesNotToBeDeleted); - Set generationsToBeDeleted = getGenerationsToBeDeleted( - metadataFilesNotToBeDeleted, - metadataFilesToBeDeleted, - indexDeleted - ); - - logger.debug("generationsToBeDeleted = {}", generationsToBeDeleted); - if (generationsToBeDeleted.isEmpty() == false) { - // Delete stale generations - translogTransferManager.deleteGenerationAsync( - primaryTermSupplier.getAsLong(), - generationsToBeDeleted, - remoteGenerationDeletionPermits::release - ); - - // Delete stale metadata files - translogTransferManager.deleteMetadataFilesAsync( - metadataFilesToBeDeleted, - remoteGenerationDeletionPermits::release - ); - - // Update cache to keep only those metadata files that are not getting deleted - oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); - - // Delete stale primary terms - deleteStaleRemotePrimaryTerms(metadataFiles); - } else { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - } - } catch (Exception e) { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - } - } - - @Override - public void onFailure(Exception e) { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); - logger.error("Exception while listing translog metadata files", e); + // cleans up remote translog files not referenced in latest uploaded metadata. + // This enables us to restore translog from the metadata in case of failover or relocation. + Set generationsToDelete = new HashSet<>(); + for (long generation = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); generation >= 0; generation--) { + if (fileTransferTracker.uploaded(Translog.getFilename(generation)) == false) { + break; } - }; - translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); - } - - // Visible for testing - protected Set getGenerationsToBeDeleted( - List metadataFilesNotToBeDeleted, - List metadataFilesToBeDeleted, - boolean indexDeleted - ) throws IOException { - long maxGenerationToBeDeleted = Long.MAX_VALUE; - - if (indexDeleted == false) { - maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); + generationsToDelete.add(generation); } - - Set generationsFromMetadataFilesToBeDeleted = new HashSet<>(); - for (String mdFile : metadataFilesToBeDeleted) { - Tuple minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager); - generationsFromMetadataFilesToBeDeleted.addAll( - LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList()) - ); - } - - Map> metadataFileNotToBeDeletedGenerationMap = getGenerationForMetadataFiles(metadataFilesNotToBeDeleted); - TreeSet> pinnedGenerations = getOrderedPinnedMetadataGenerations(metadataFileNotToBeDeletedGenerationMap); - Set generationsToBeDeleted = new HashSet<>(); - for (long generation : generationsFromMetadataFilesToBeDeleted) { - // Check if the generation is not referred by metadata file matching pinned timestamps - if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) { - generationsToBeDeleted.add(generation); - } + if (generationsToDelete.isEmpty() == false) { + deleteRemoteGeneration(generationsToDelete); + translogTransferManager.deleteStaleTranslogMetadataFilesAsync(remoteGenerationDeletionPermits::release); + deleteStaleRemotePrimaryTerms(); + } else { + remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); } - return generationsToBeDeleted; } - // Visible for testing - protected List getMetadataFilesToBeDeleted(List metadataFiles) { - Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); - - // Keep files since last successful run of scheduler - List metadataFilesToBeDeleted = RemoteStoreUtils.filterOutMetadataFilesBasedOnAge( - metadataFiles, - file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), - pinnedTimestampsState.v1() - ); - - logger.trace( - "metadataFiles.size = {}, metadataFilesToBeDeleted based on age based filtering = {}", - metadataFiles.size(), - metadataFilesToBeDeleted.size() - ); - - // Get md files matching pinned timestamps - Set implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles( - metadataFilesToBeDeleted, - pinnedTimestampsState.v2(), - metadataFilePinnedTimestampMap, - file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), - TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen - ); - - // Filter out metadata files matching pinned timestamps - metadataFilesToBeDeleted.removeAll(implicitLockedFiles); - - logger.trace( - "implicitLockedFiles.size = {}, metadataFilesToBeDeleted based on pinned timestamp filtering = {}", - implicitLockedFiles.size(), - metadataFilesToBeDeleted.size() + /** + * Deletes remote translog and metadata files asynchronously corresponding to the generations. + * @param generations generations to be deleted. + */ + private void deleteRemoteGeneration(Set generations) { + translogTransferManager.deleteGenerationAsync( + primaryTermSupplier.getAsLong(), + generations, + remoteGenerationDeletionPermits::release ); - - return metadataFilesToBeDeleted; - } - - // Visible for testing - protected boolean isGenerationPinned(long generation, TreeSet> pinnedGenerations) { - Tuple ceilingGenerationRange = pinnedGenerations.ceiling(new Tuple<>(generation, generation)); - if (ceilingGenerationRange != null && generation >= ceilingGenerationRange.v1() && generation <= ceilingGenerationRange.v2()) { - return true; - } - Tuple floorGenerationRange = pinnedGenerations.floor(new Tuple<>(generation, generation)); - if (floorGenerationRange != null && generation >= floorGenerationRange.v1() && generation <= floorGenerationRange.v2()) { - return true; - } - return false; - } - - private TreeSet> getOrderedPinnedMetadataGenerations(Map> metadataFileGenerationMap) { - TreeSet> pinnedGenerations = new TreeSet<>((o1, o2) -> { - if (Objects.equals(o1.v1(), o2.v1()) == false) { - return o1.v1().compareTo(o2.v1()); - } else { - return o1.v2().compareTo(o2.v2()); - } - }); - pinnedGenerations.addAll(metadataFileGenerationMap.values()); - return pinnedGenerations; - } - - // Visible for testing - protected Map> getGenerationForMetadataFiles(List metadataFiles) throws IOException { - Map> metadataFileGenerationMap = new HashMap<>(); - for (String metadataFile : metadataFiles) { - metadataFileGenerationMap.put(metadataFile, getMinMaxTranslogGenerationFromMetadataFile(metadataFile, translogTransferManager)); - } - return metadataFileGenerationMap; - } - - // Visible for testing - protected Tuple getMinMaxTranslogGenerationFromMetadataFile( - String metadataFile, - TranslogTransferManager translogTransferManager - ) throws IOException { - Tuple minMaxGenerationFromFileName = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(metadataFile); - if (minMaxGenerationFromFileName != null) { - return minMaxGenerationFromFileName; - } else { - if (oldFormatMetadataFileGenerationMap.containsKey(metadataFile)) { - return oldFormatMetadataFileGenerationMap.get(metadataFile); - } else { - TranslogTransferMetadata metadata = translogTransferManager.readMetadata(metadataFile); - Tuple minMaxGenTuple = new Tuple<>(metadata.getMinTranslogGeneration(), metadata.getGeneration()); - oldFormatMetadataFileGenerationMap.put(metadataFile, minMaxGenTuple); - return minMaxGenTuple; - } - } } /** @@ -825,20 +601,17 @@ protected Tuple getMinMaxTranslogGenerationFromMetadataFile( *
* This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ - private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + private void deleteStaleRemotePrimaryTerms() { // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. if (olderPrimaryCleaned.trySet(Boolean.TRUE)) { - if (metadataFiles.isEmpty()) { - logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); + if (readers.isEmpty()) { + logger.trace("Translog reader list is empty, returning from deleteStaleRemotePrimaryTerms"); return; } - Optional minPrimaryTerm = metadataFiles.stream() - .map(file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[1])) - .min(Long::compareTo); // First we delete all stale primary terms folders from remote store - long minimumReferencedPrimaryTerm = minPrimaryTerm.get() - 1; + long minimumReferencedPrimaryTerm = readers.stream().map(BaseTranslogReader::getPrimaryTerm).min(Long::compare).get(); translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); } } @@ -874,15 +647,7 @@ public static void cleanup( protected void onDelete() { ClusterService.assertClusterOrClusterManagerStateThread(); // clean up all remote translog files - if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { - try { - trimUnreferencedReaders(true, false); - } catch (IOException e) { - logger.error("Exception while deleting translog files from remote store", e); - } - } else { - translogTransferManager.delete(); - } + translogTransferManager.delete(); } // Visible for testing diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java similarity index 93% rename from server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java rename to server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 386dde4dffc48..1f82dd9d7e641 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogWithPinnedTimestampTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -29,6 +29,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.translog.transfer.TranslogTransferManager; import org.opensearch.index.translog.transfer.TranslogTransferMetadata; +import org.opensearch.index.translog.transfer.TranslogUploadFailedException; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.Node; @@ -42,6 +43,7 @@ import org.junit.Before; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -60,14 +62,13 @@ import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; -import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @LuceneTestCase.SuppressFileSystems("ExtrasFS") -public class RemoteFsTranslogWithPinnedTimestampTests extends RemoteFsTranslogTests { +public class RemoteFsTimestampAwareTranslogTests extends RemoteFsTranslogTests { Runnable updatePinnedTimstampTask; BlobContainer blobContainer; @@ -125,6 +126,48 @@ public void setUp() throws Exception { remoteStorePinnedTimestampServiceSpy.start(); } + @Override + protected RemoteFsTranslog createTranslogInstance( + TranslogConfig translogConfig, + String translogUUID, + TranslogDeletionPolicy deletionPolicy + ) throws IOException { + return new RemoteFsTimestampAwareTranslog( + translogConfig, + translogUUID, + deletionPolicy, + () -> globalCheckpoint.get(), + primaryTerm::get, + getPersistedSeqNoConsumer(), + repository, + threadPool, + primaryMode::get, + new RemoteTranslogTransferTracker(shardId, 10), + DefaultRemoteStoreSettings.INSTANCE + ); + } + + @Override + public void testSyncUpAlwaysFailure() throws IOException { + int translogOperations = randomIntBetween(1, 20); + int count = 0; + fail.failAlways(); + for (int op = 0; op < translogOperations; op++) { + translog.add( + new Translog.Index(String.valueOf(op), count, primaryTerm.get(), Integer.toString(count).getBytes(StandardCharsets.UTF_8)) + ); + try { + translog.sync(); + fail("io exception expected"); + } catch (TranslogUploadFailedException e) { + assertTrue("at least one operation pending", translog.syncNeeded()); + } + } + assertTrue(translog.isOpen()); + fail.failNever(); + translog.sync(); + } + public void testGetMinMaxTranslogGenerationFromFilename() throws Exception { RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); ArrayList ops = new ArrayList<>(); @@ -198,7 +241,7 @@ public void testIndexDeletionWithNoPinnedTimestampNoRecentMdFiles() throws Excep assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); updatePinnedTimstampTask.run(); - translog.trimUnreferencedReaders(true, false); + ((RemoteFsTimestampAwareTranslog) translog).trimUnreferencedReaders(true, false); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); @@ -221,7 +264,7 @@ public void testIndexDeletionWithNoPinnedTimestampButRecentFiles() throws Except addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); updatePinnedTimstampTask.run(); - translog.trimUnreferencedReaders(true, false); + ((RemoteFsTimestampAwareTranslog) translog).trimUnreferencedReaders(true, false); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertBusy(() -> { @@ -567,7 +610,11 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro // 27 to 42 "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" ); - Set generations = translog.getGenerationsToBeDeleted(metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, true); + Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + true + ); Set md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet()); Set md3Generations = LongStream.rangeClosed(27, 42).boxed().collect(Collectors.toSet()); @@ -598,7 +645,11 @@ public void testGetGenerationsToBeDeleted() throws IOException { // 27 to 42 "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" ); - Set generations = translog.getGenerationsToBeDeleted(metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, true); + Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + true + ); Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); Set md3Generations = LongStream.rangeClosed(31, 41).boxed().collect(Collectors.toSet()); @@ -621,7 +672,7 @@ public void testGetMetadataFilesToBeDeletedNoExclusion() { "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1" ); - assertEquals(metadataFiles, translog.getMetadataFilesToBeDeleted(metadataFiles)); + assertEquals(metadataFiles, ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles)); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { @@ -637,7 +688,7 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); } @@ -659,7 +710,7 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); assertEquals(2, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); @@ -682,7 +733,7 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throw "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = translog.getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0)); } @@ -707,6 +758,8 @@ public void testIsGenerationPinned() { pinnedGenerations.add(new Tuple<>(142L, 180L)); pinnedGenerations.add(new Tuple<>(4L, 9L)); + RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog; + assertFalse(translog.isGenerationPinned(3, pinnedGenerations)); assertFalse(translog.isGenerationPinned(10, pinnedGenerations)); assertFalse(translog.isGenerationPinned(141, pinnedGenerations)); @@ -724,6 +777,8 @@ public void testIsGenerationPinned() { public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException { TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog; + // Fetch generations directly from the filename assertEquals( new Tuple<>(701L, 1008L), diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 163340e8ec7d5..339d876274557 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -118,11 +118,11 @@ public class RemoteFsTranslogTests extends OpenSearchTestCase { protected final ShardId shardId = new ShardId("index", "_na_", 1); protected RemoteFsTranslog translog; - private AtomicLong globalCheckpoint; + protected AtomicLong globalCheckpoint; protected Path translogDir; // A default primary term is used by translog instances created in this test. protected final AtomicLong primaryTerm = new AtomicLong(); - private final AtomicBoolean primaryMode = new AtomicBoolean(true); + protected final AtomicBoolean primaryMode = new AtomicBoolean(true); private final AtomicReference persistedSeqNoConsumer = new AtomicReference<>(); protected ThreadPool threadPool; protected final static String METADATA_DIR = "metadata"; @@ -136,7 +136,7 @@ public class RemoteFsTranslogTests extends OpenSearchTestCase { TestTranslog.SlowDownWriteSwitch slowDown; - private LongConsumer getPersistedSeqNoConsumer() { + protected LongConsumer getPersistedSeqNoConsumer() { return seqNo -> { final LongConsumer consumer = persistedSeqNoConsumer.get(); if (consumer != null) { @@ -167,7 +167,7 @@ public void tearDown() throws Exception { } } - private RemoteFsTranslog create(Path path) throws IOException { + protected RemoteFsTranslog create(Path path) throws IOException { final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get()); return create(path, createRepository(), translogUUID, 0); } @@ -179,6 +179,14 @@ private RemoteFsTranslog create(Path path, BlobStoreRepository repository, Strin final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings()); threadPool = new TestThreadPool(getClass().getName()); blobStoreTransferService = new BlobStoreTransferService(repository.blobStore(), threadPool); + return createTranslogInstance(translogConfig, translogUUID, deletionPolicy); + } + + protected RemoteFsTranslog createTranslogInstance( + TranslogConfig translogConfig, + String translogUUID, + TranslogDeletionPolicy deletionPolicy + ) throws IOException { return new RemoteFsTranslog( translogConfig, translogUUID, @@ -671,13 +679,13 @@ public void testSimpleOperationsUpload() throws Exception { assertThat(snapshot.totalOperations(), equalTo(ops.size())); } - assertEquals(2, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); - assertEquals(4, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); translog.rollGeneration(); - assertEquals(4, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); Set mdFiles = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); assertEquals(2, mdFiles.size()); @@ -736,7 +744,7 @@ public void testSimpleOperationsUpload() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); assertBusy(() -> { - assertEquals(2, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); assertEquals( 4, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() @@ -755,7 +763,7 @@ public void testSimpleOperationsUpload() throws Exception { assertEquals(1, translog.readers.size()); assertEquals(1, translog.stats().estimatedNumberOfOperations()); assertBusy(() -> { - assertEquals(2, translog.allUploaded().size()); + assertEquals(4, translog.allUploaded().size()); assertEquals( 4, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() @@ -774,7 +782,7 @@ public void testMetadataFileDeletion() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); } - assertBusy(() -> assertEquals(2, translog.allUploaded().size())); + assertBusy(() -> assertEquals(4, translog.allUploaded().size())); assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); int moreDocs = randomIntBetween(3, 10); logger.info("numDocs={} moreDocs={}", numDocs, moreDocs); @@ -872,7 +880,7 @@ public void testDrainSync() throws Exception { assertBusy(() -> assertEquals(0, latch.getCount())); assertEquals(0, translog.availablePermits()); slowDown.setSleepSeconds(0); - assertEquals(4, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); assertEquals(2, translog.readers.size()); Set mdFiles = blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)); @@ -881,7 +889,7 @@ public void testDrainSync() throws Exception { translog.trimUnreferencedReaders(); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); - assertEquals(2, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); // Case 4 - After drainSync, if an upload is an attempted, we do not upload to remote store. @@ -891,21 +899,21 @@ public void testDrainSync() throws Exception { new Translog.Index(String.valueOf(2), 2, primaryTerm.get(), new byte[] { 1 }) ); assertEquals(1, translog.readers.size()); - assertEquals(2, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); assertEquals(mdFiles, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR))); // Refill the permits back Releasables.close(releasable); addToTranslogAndListAndUpload(translog, ops, new Translog.Index(String.valueOf(3), 3, primaryTerm.get(), new byte[] { 1 })); assertEquals(2, translog.readers.size()); - assertEquals(4, translog.allUploaded().size()); + assertEquals(8, translog.allUploaded().size()); assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); translog.setMinSeqNoToKeep(3); translog.trimUnreferencedReaders(); assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); - assertBusy(() -> assertEquals(2, translog.allUploaded().size())); + assertBusy(() -> assertEquals(4, translog.allUploaded().size())); assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); } From 3fc0139ca68a1ff843ec1492c3cd52c2c4c67f02 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Tue, 3 Sep 2024 23:42:26 +0530 Subject: [PATCH 028/357] Optimise snapshot deletion to speed up snapshot deletion and creation (#15568) --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../blobstore/BlobStoreRepository.java | 92 ++++++++++++++++--- .../org/opensearch/threadpool/ThreadPool.java | 7 ++ .../threadpool/ScalingThreadPoolTests.java | 1 + 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93a98de2c5af8..4df028f12d2fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) +- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index d193ab3c14154..aff48f717eb44 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -66,6 +66,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Numbers; import org.opensearch.common.Priority; +import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobContainer; @@ -831,7 +832,7 @@ boolean getPrefixModeVerification() { * maintains single lazy instance of {@link BlobContainer} */ protected BlobContainer blobContainer() { - // assertSnapshotOrGenericThread(); + assertSnapshotOrGenericThread(); BlobContainer blobContainer = this.blobContainer.get(); if (blobContainer == null) { @@ -1204,6 +1205,10 @@ private void asyncCleanupUnlinkedShardLevelBlobs( ActionListener listener ) { final List> filesToDelete = resolveFilesToDelete(oldRepositoryData, snapshotIds, deleteResults); + long startTimeNs = System.nanoTime(); + Randomness.shuffle(filesToDelete); + logger.debug("[{}] shuffled the filesToDelete with timeElapsedNs={}", metadata.name(), (System.nanoTime() - startTimeNs)); + if (filesToDelete.isEmpty()) { listener.onResponse(null); return; @@ -1221,8 +1226,8 @@ private void asyncCleanupUnlinkedShardLevelBlobs( staleFilesToDeleteInBatch.size() ); - // Start as many workers as fit into the snapshot pool at once at the most - final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), staleFilesToDeleteInBatch.size()); + // Start as many workers as fit into the snapshot_deletion pool at once at the most + final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT_DELETION).getMax(), staleFilesToDeleteInBatch.size()); for (int i = 0; i < workers; ++i) { executeStaleShardDelete(staleFilesToDeleteInBatch, remoteStoreLockManagerFactory, groupedListener); } @@ -1326,7 +1331,7 @@ private void executeStaleShardDelete( if (filesToDelete == null) { return; } - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { + threadPool.executor(ThreadPool.Names.SNAPSHOT_DELETION).execute(ActionRunnable.wrap(listener, l -> { try { // filtering files for which remote store lock release and cleanup succeeded, // remaining files for which it failed will be retried in next snapshot delete run. @@ -1390,7 +1395,7 @@ private void writeUpdatedShardMetaDataAndComputeDeletes( ActionListener> onAllShardsCompleted ) { - final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT_DELETION); final List indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); if (indices.isEmpty()) { @@ -1578,7 +1583,7 @@ private void cleanupStaleBlobs( listener.onResponse(deleteResult); }, listener::onFailure), 2); - final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT_DELETION); final List staleRootBlobs = staleRootBlobs(newRepoData, rootBlobs.keySet()); if (staleRootBlobs.isEmpty()) { groupedListener.onResponse(DeleteResult.ZERO); @@ -1781,7 +1786,7 @@ void cleanupStaleIndices( // Start as many workers as fit into the snapshot pool at once at the most final int workers = Math.min( - threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), + threadPool.info(ThreadPool.Names.SNAPSHOT_DELETION).getMax(), foundIndices.size() - survivingIndexIds.size() ); for (int i = 0; i < workers; ++i) { @@ -1833,7 +1838,7 @@ private void executeOneStaleIndexDelete( return; } final String indexSnId = indexEntry.getKey(); - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(listener, () -> { + threadPool.executor(ThreadPool.Names.SNAPSHOT_DELETION).execute(ActionRunnable.supply(listener, () -> { try { logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId); List matchingShardPaths = findMatchingShardPaths(indexSnId, snapshotShardPaths); @@ -2097,8 +2102,7 @@ public void finalizeSnapshot( stateTransformer, repositoryUpdatePriority, ActionListener.wrap(newRepoData -> { - cleanupOldShardGens(existingRepositoryData, updatedRepositoryData); - listener.onResponse(newRepoData); + cleanupOldShardGens(existingRepositoryData, updatedRepositoryData, newRepoData, listener); }, onUpdateFailure) ); }, onUpdateFailure), 2 + indices.size()); @@ -2254,7 +2258,12 @@ private void logShardPathsOperationWarning(IndexId indexId, SnapshotId snapshotI } // Delete all old shard gen blobs that aren't referenced any longer as a result from moving to updated repository data - private void cleanupOldShardGens(RepositoryData existingRepositoryData, RepositoryData updatedRepositoryData) { + private void cleanupOldShardGens( + RepositoryData existingRepositoryData, + RepositoryData updatedRepositoryData, + RepositoryData newRepositoryData, + ActionListener listener + ) { final List toDelete = new ArrayList<>(); updatedRepositoryData.shardGenerations() .obsoleteShardGenerations(existingRepositoryData.shardGenerations()) @@ -2263,10 +2272,62 @@ private void cleanupOldShardGens(RepositoryData existingRepositoryData, Reposito (shardId, oldGen) -> toDelete.add(shardPath(indexId, shardId).buildAsString() + INDEX_FILE_PREFIX + oldGen) ) ); + if (toDelete.isEmpty()) { + listener.onResponse(newRepositoryData); + return; + } try { - deleteFromContainer(rootBlobContainer(), toDelete); + AtomicInteger counter = new AtomicInteger(); + Collection> subList = toDelete.stream() + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / maxShardBlobDeleteBatch)) + .values(); + final BlockingQueue> staleFilesToDeleteInBatch = new LinkedBlockingQueue<>(subList); + logger.info( + "[{}] cleanupOldShardGens toDeleteSize={} groupSize={}", + metadata.name(), + toDelete.size(), + staleFilesToDeleteInBatch.size() + ); + final GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(r -> { + logger.info("[{}] completed cleanupOldShardGens", metadata.name()); + listener.onResponse(newRepositoryData); + }, ex -> { + logger.error(new ParameterizedMessage("[{}] exception in cleanupOldShardGens", metadata.name()), ex); + listener.onResponse(newRepositoryData); + }), staleFilesToDeleteInBatch.size()); + + // Start as many workers as fit into the snapshot pool at once at the most + final int workers = Math.min(threadPool.info(ThreadPool.Names.SNAPSHOT_DELETION).getMax(), staleFilesToDeleteInBatch.size()); + for (int i = 0; i < workers; ++i) { + executeOldShardGensCleanup(staleFilesToDeleteInBatch, groupedListener); + } } catch (Exception e) { - logger.warn("Failed to clean up old shard generation blobs", e); + logger.warn(new ParameterizedMessage(" [{}] Failed to clean up old shard generation blobs", metadata.name()), e); + listener.onResponse(newRepositoryData); + } + } + + private void executeOldShardGensCleanup(BlockingQueue> staleFilesToDeleteInBatch, GroupedActionListener listener) + throws InterruptedException { + List filesToDelete = staleFilesToDeleteInBatch.poll(0L, TimeUnit.MILLISECONDS); + if (filesToDelete != null) { + threadPool.executor(ThreadPool.Names.SNAPSHOT_DELETION).execute(ActionRunnable.wrap(listener, l -> { + try { + deleteFromContainer(rootBlobContainer(), filesToDelete); + l.onResponse(null); + } catch (Exception e) { + logger.warn( + () -> new ParameterizedMessage( + "[{}] Failed to delete following blobs during cleanupOldFiles : {}", + metadata.name(), + filesToDelete + ), + e + ); + l.onFailure(e); + } + executeOldShardGensCleanup(staleFilesToDeleteInBatch, listener); + })); } } @@ -2383,10 +2444,11 @@ public long getRemoteDownloadThrottleTimeInNanos() { } protected void assertSnapshotOrGenericThread() { - assert Thread.currentThread().getName().contains('[' + ThreadPool.Names.SNAPSHOT + ']') + assert Thread.currentThread().getName().contains('[' + ThreadPool.Names.SNAPSHOT_DELETION + ']') + || Thread.currentThread().getName().contains('[' + ThreadPool.Names.SNAPSHOT + ']') || Thread.currentThread().getName().contains('[' + ThreadPool.Names.GENERIC + ']') : "Expected current thread [" + Thread.currentThread() - + "] to be the snapshot or generic thread."; + + "] to be the snapshot_deletion or snapshot or generic thread."; } @Override diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 056ef0fac0153..81220ab171b34 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -105,6 +105,7 @@ public static class Names { public static final String REFRESH = "refresh"; public static final String WARMER = "warmer"; public static final String SNAPSHOT = "snapshot"; + public static final String SNAPSHOT_DELETION = "snapshot_deletion"; public static final String FORCE_MERGE = "force_merge"; public static final String FETCH_SHARD_STARTED = "fetch_shard_started"; public static final String FETCH_SHARD_STORE = "fetch_shard_store"; @@ -176,6 +177,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.REFRESH, ThreadPoolType.SCALING); map.put(Names.WARMER, ThreadPoolType.SCALING); map.put(Names.SNAPSHOT, ThreadPoolType.SCALING); + map.put(Names.SNAPSHOT_DELETION, ThreadPoolType.SCALING); map.put(Names.FORCE_MERGE, ThreadPoolType.FIXED); map.put(Names.FETCH_SHARD_STARTED, ThreadPoolType.SCALING); map.put(Names.FETCH_SHARD_STORE, ThreadPoolType.SCALING); @@ -234,6 +236,7 @@ public ThreadPool( final int halfProcMaxAt5 = halfAllocatedProcessorsMaxFive(allocatedProcessors); final int halfProcMaxAt10 = halfAllocatedProcessorsMaxTen(allocatedProcessors); final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512); + final int snapshotDeletionPoolMax = boundedBy(4 * allocatedProcessors, 64, 256); builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30))); builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000)); @@ -251,6 +254,10 @@ public ThreadPool( builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5))); builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5))); + builders.put( + Names.SNAPSHOT_DELETION, + new ScalingExecutorBuilder(Names.SNAPSHOT_DELETION, 1, snapshotDeletionPoolMax, TimeValue.timeValueMinutes(5)) + ); builders.put( Names.FETCH_SHARD_STARTED, new ScalingExecutorBuilder(Names.FETCH_SHARD_STARTED, 1, 2 * allocatedProcessors, TimeValue.timeValueMinutes(5)) diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index d8f04a11fe494..4f59f9688fb7e 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -148,6 +148,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.REFRESH, ThreadPool::halfAllocatedProcessorsMaxTen); sizes.put(ThreadPool.Names.WARMER, ThreadPool::halfAllocatedProcessorsMaxFive); sizes.put(ThreadPool.Names.SNAPSHOT, ThreadPool::halfAllocatedProcessorsMaxFive); + sizes.put(ThreadPool.Names.SNAPSHOT_DELETION, n -> ThreadPool.boundedBy(4 * n, 64, 256)); sizes.put(ThreadPool.Names.FETCH_SHARD_STARTED, ThreadPool::twiceAllocatedProcessors); sizes.put(ThreadPool.Names.FETCH_SHARD_STORE, ThreadPool::twiceAllocatedProcessors); sizes.put(ThreadPool.Names.TRANSLOG_TRANSFER, ThreadPool::halfAllocatedProcessors); From a60b668ad98a52ee8e42ace52de3f68caff36276 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Wed, 4 Sep 2024 00:54:44 +0530 Subject: [PATCH 029/357] Remove breaking changes from #15409 (#15624) Signed-off-by: Lakshya Taragi --- .../client/SnapshotRequestConvertersTests.java | 2 +- .../upgrades/MultiVersionRepositoryAccessIT.java | 2 +- .../snapshots/status/SnapshotsStatusRequest.java | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java index e9ca03262bc4f..8b6910ffebe4a 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java @@ -238,7 +238,7 @@ public void testSnapshotsStatus() { boolean ignoreUnavailable = randomBoolean(); String endpoint = "/_snapshot/" + repository + "/" + snapshotNames.toString() + "/_status"; - SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots, indices); + SnapshotsStatusRequest snapshotsStatusRequest = (new SnapshotsStatusRequest(repository, snapshots)).indices(indices); RequestConvertersTests.setRandomClusterManagerTimeout(snapshotsStatusRequest, expectedParams); snapshotsStatusRequest.ignoreUnavailable(ignoreUnavailable); expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable)); diff --git a/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java b/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java index 5c6dcdd142995..c47e0065b708b 100644 --- a/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java +++ b/qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java @@ -243,7 +243,7 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException { private static void assertSnapshotStatusSuccessful(RestHighLevelClient client, String repoName, String[] snapshots, String[] indices) throws IOException { final SnapshotsStatusResponse statusResponse = client.snapshot() - .status(new SnapshotsStatusRequest(repoName, snapshots, indices), RequestOptions.DEFAULT); + .status((new SnapshotsStatusRequest(repoName, snapshots)).indices(indices), RequestOptions.DEFAULT); for (SnapshotStatus status : statusResponse.getSnapshots()) { assertThat(status.getShardsStats().getFailedShards(), is(0)); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index 6c3438fbe58f6..3d7fb5b6beb56 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -67,6 +67,18 @@ public SnapshotsStatusRequest() {} * @param repository repository name * @param snapshots list of snapshots */ + public SnapshotsStatusRequest(String repository, String[] snapshots) { + this.repository = repository; + this.snapshots = snapshots; + } + + /** + * Constructs a new get snapshots request with given repository name and list of snapshots + * + * @param repository repository name + * @param snapshots list of snapshots + * @param indices list of indices + */ public SnapshotsStatusRequest(String repository, String[] snapshots, String[] indices) { this.repository = repository; this.snapshots = snapshots; From c19cf88d266d15d6e8c69296cd0626099e562e38 Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Tue, 3 Sep 2024 13:54:16 -0700 Subject: [PATCH 030/357] Add canRemain method to TargetPoolAllocationDecider to move shards from (#15010) local to remote pool for hot to warm tiering Signed-off-by: Neetika Singhal --- CHANGELOG.md | 1 + .../admin/indices/tiering/TieringUtils.java | 47 +++++++ .../cluster/routing/RoutingPool.java | 6 +- .../allocator/LocalShardsBalancer.java | 20 ++- .../decider/TargetPoolAllocationDecider.java | 30 ++++ .../ShardsTieringAllocationTests.java | 128 ++++++++++++++++++ .../TieringAllocationBaseTestCase.java | 47 +++++++ 7 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df028f12d2fe..a76bba98cd902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) - Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) +- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java new file mode 100644 index 0000000000000..46912de17f213 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringUtils.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.tiering; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.index.IndexModule; + +/** + * Utility class for tiering operations + * + * @opensearch.internal + */ +public class TieringUtils { + + /** + * Checks if the specified shard is a partial shard by + * checking the INDEX_STORE_LOCALITY_SETTING for its index. + * see {@link #isPartialIndex(IndexMetadata)} + * @param shard ShardRouting object representing the shard + * @param allocation RoutingAllocation object representing the allocation + * @return true if the shard is a partial shard, false otherwise + */ + public static boolean isPartialShard(ShardRouting shard, RoutingAllocation allocation) { + IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shard.index()); + return isPartialIndex(indexMetadata); + } + + /** + * Checks if the specified index is a partial index by + * checking the INDEX_STORE_LOCALITY_SETTING for the index. + * + * @param indexMetadata the metadata of the index + * @return true if the index is a partial index, false otherwise + */ + public static boolean isPartialIndex(final IndexMetadata indexMetadata) { + return IndexModule.DataLocalityType.PARTIAL.name() + .equals(indexMetadata.getSettings().get(IndexModule.INDEX_STORE_LOCALITY_SETTING.getKey())); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java index db10ad61c7d6d..647e993339476 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingPool.java @@ -11,6 +11,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.common.util.FeatureFlags; + +import static org.opensearch.action.admin.indices.tiering.TieringUtils.isPartialIndex; /** * {@link RoutingPool} defines the different node types based on the assigned capabilities. The methods @@ -58,6 +61,7 @@ public static RoutingPool getShardPool(ShardRouting shard, RoutingAllocation all * @return {@link RoutingPool} for the given index. */ public static RoutingPool getIndexPool(IndexMetadata indexMetadata) { - return indexMetadata.isRemoteSnapshot() ? REMOTE_CAPABLE : LOCAL_ONLY; + return indexMetadata.isRemoteSnapshot() + || (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialIndex(indexMetadata)) ? REMOTE_CAPABLE : LOCAL_ONLY; } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java index adb8ee2cf7e85..7f6a7790d1db0 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java @@ -30,6 +30,7 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.cluster.routing.allocation.decider.DiskThresholdDecider; import org.opensearch.common.collect.Tuple; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.PriorityComparator; import java.util.ArrayList; @@ -45,6 +46,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.opensearch.action.admin.indices.tiering.TieringUtils.isPartialShard; import static org.opensearch.cluster.routing.ShardRoutingState.RELOCATING; /** @@ -552,6 +554,16 @@ private void checkAndAddInEligibleTargetNode(RoutingNode targetNode) { } } + /** + * Checks if the shard can be skipped from the local shard balancer operations + * @param shardRouting the shard to be checked + * @return true if the shard can be skipped, false otherwise + */ + private boolean canShardBeSkipped(ShardRouting shardRouting) { + return (RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shardRouting, allocation)) + && !(FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialShard(shardRouting, allocation))); + } + /** * Move started shards that can not be allocated to a node anymore *

@@ -603,7 +615,7 @@ void moveShards() { ShardRouting shardRouting = it.next(); - if (RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shardRouting, allocation))) { + if (canShardBeSkipped(shardRouting)) { continue; } @@ -669,7 +681,7 @@ void moveShards() { */ @Override MoveDecision decideMove(final ShardRouting shardRouting) { - if (RoutingPool.REMOTE_CAPABLE.equals(RoutingPool.getShardPool(shardRouting, allocation))) { + if (canShardBeSkipped(shardRouting)) { return MoveDecision.NOT_TAKEN; } @@ -758,7 +770,9 @@ private Map buildModelFromAssigned() for (ShardRouting shard : rn) { assert rn.nodeId().equals(shard.currentNodeId()); /* we skip relocating shards here since we expect an initializing shard with the same id coming in */ - if (RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)) && shard.state() != RELOCATING) { + if ((RoutingPool.LOCAL_ONLY.equals(RoutingPool.getShardPool(shard, allocation)) + || (FeatureFlags.isEnabled(FeatureFlags.TIERED_REMOTE_INDEX) && isPartialShard(shard, allocation))) + && shard.state() != RELOCATING) { node.addShard(shard); ++totalShardCount; if (logger.isTraceEnabled()) { diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/TargetPoolAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/TargetPoolAllocationDecider.java index 76f9f44077ad8..493d23b57d271 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/TargetPoolAllocationDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/TargetPoolAllocationDecider.java @@ -87,6 +87,36 @@ public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode n return canAllocate(shardRouting, node, allocation); } + @Override + public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + RoutingPool targetPool = RoutingPool.getShardPool(shardRouting, allocation); + RoutingPool currentNodePool = RoutingPool.getNodePool(allocation.routingNodes().node(shardRouting.currentNodeId())); + if (RoutingPool.REMOTE_CAPABLE.equals(targetPool) && targetPool != currentNodePool) { + logger.debug( + "Shard: [{}] has current pool: [{}], target pool: [{}]. Cannot remain on node: [{}]", + shardRouting.shortSummary(), + currentNodePool.name(), + RoutingPool.REMOTE_CAPABLE.name(), + node.node() + ); + return allocation.decision( + Decision.NO, + NAME, + "Shard %s is allocated on a different pool %s than the target pool %s", + shardRouting.shortSummary(), + currentNodePool, + targetPool + ); + } + return allocation.decision( + Decision.YES, + NAME, + "Routing pools are compatible. Shard pool: [%s], node pool: [%s]", + currentNodePool, + targetPool + ); + } + public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) { logger.debug("Evaluating node: {} for autoExpandReplica eligibility of index: {}", node, indexMetadata.getIndex()); return canAllocateInTargetPool(indexMetadata, node, allocation); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java new file mode 100644 index 0000000000000..8d45ebd2781b1 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/ShardsTieringAllocationTests.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.RoutingNode; +import org.opensearch.cluster.routing.RoutingNodes; +import org.opensearch.cluster.routing.RoutingPool; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexModule; +import org.opensearch.test.FeatureFlagSetter; +import org.junit.Before; + +import static org.opensearch.cluster.routing.RoutingPool.LOCAL_ONLY; +import static org.opensearch.cluster.routing.RoutingPool.REMOTE_CAPABLE; +import static org.opensearch.cluster.routing.RoutingPool.getIndexPool; +import static org.opensearch.index.IndexModule.INDEX_STORE_LOCALITY_SETTING; + +public class ShardsTieringAllocationTests extends TieringAllocationBaseTestCase { + + @Before + public void setup() { + FeatureFlagSetter.set(FeatureFlags.TIERED_REMOTE_INDEX); + } + + public void testShardsInLocalPool() { + int localOnlyNodes = 5; + int remoteCapableNodes = 3; + int localIndices = 5; + int remoteIndices = 0; + ClusterState clusterState = createInitialCluster(localOnlyNodes, remoteCapableNodes, localIndices, remoteIndices); + AllocationService service = this.createRemoteCapableAllocationService(); + // assign shards to respective nodes + clusterState = allocateShardsAndBalance(clusterState, service); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + RoutingAllocation allocation = getRoutingAllocation(clusterState, routingNodes); + assertEquals(0, routingNodes.unassigned().size()); + + for (ShardRouting shard : clusterState.getRoutingTable().allShards()) { + assertFalse(shard.unassigned()); + RoutingPool shardPool = RoutingPool.getShardPool(shard, allocation); + assertEquals(LOCAL_ONLY, shardPool); + } + } + + public void testShardsInRemotePool() { + int localOnlyNodes = 7; + int remoteCapableNodes = 3; + int localIndices = 0; + int remoteIndices = 13; + ClusterState clusterState = createInitialCluster(localOnlyNodes, remoteCapableNodes, localIndices, remoteIndices); + AllocationService service = this.createRemoteCapableAllocationService(); + // assign shards to respective nodes + clusterState = allocateShardsAndBalance(clusterState, service); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + RoutingAllocation allocation = getRoutingAllocation(clusterState, routingNodes); + assertEquals(0, routingNodes.unassigned().size()); + + for (ShardRouting shard : clusterState.getRoutingTable().allShards()) { + assertFalse(shard.unassigned()); + RoutingPool shardPool = RoutingPool.getShardPool(shard, allocation); + assertEquals(REMOTE_CAPABLE, shardPool); + } + } + + public void testShardsWithTiering() { + int localOnlyNodes = 15; + int remoteCapableNodes = 13; + int localIndices = 10; + int remoteIndices = 0; + ClusterState clusterState = createInitialCluster(localOnlyNodes, remoteCapableNodes, localIndices, remoteIndices); + AllocationService service = this.createRemoteCapableAllocationService(); + // assign shards to respective nodes + clusterState = allocateShardsAndBalance(clusterState, service); + // put indices in the hot to warm tiering state + clusterState = updateIndexMetadataForTiering( + clusterState, + localIndices, + IndexModule.TieringState.HOT_TO_WARM.name(), + IndexModule.DataLocalityType.PARTIAL.name() + ); + // trigger shard relocation + clusterState = allocateShardsAndBalance(clusterState, service); + RoutingNodes routingNodes = clusterState.getRoutingNodes(); + RoutingAllocation allocation = getRoutingAllocation(clusterState, routingNodes); + assertEquals(0, routingNodes.unassigned().size()); + + for (ShardRouting shard : clusterState.getRoutingTable().allShards()) { + assertFalse(shard.unassigned()); + RoutingNode node = routingNodes.node(shard.currentNodeId()); + RoutingPool nodePool = RoutingPool.getNodePool(node); + RoutingPool shardPool = RoutingPool.getShardPool(shard, allocation); + assertEquals(RoutingPool.REMOTE_CAPABLE, shardPool); + assertEquals(nodePool, shardPool); + } + } + + public void testShardPoolForPartialIndices() { + String index = "test-index"; + IndexMetadata indexMetadata = IndexMetadata.builder(index) + .settings(settings(Version.CURRENT).put(INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.PARTIAL.name())) + .numberOfShards(PRIMARIES) + .numberOfReplicas(REPLICAS) + .build(); + RoutingPool indexPool = getIndexPool(indexMetadata); + assertEquals(REMOTE_CAPABLE, indexPool); + } + + public void testShardPoolForFullIndices() { + String index = "test-index"; + IndexMetadata indexMetadata = IndexMetadata.builder(index) + .settings(settings(Version.CURRENT).put(INDEX_STORE_LOCALITY_SETTING.getKey(), IndexModule.DataLocalityType.FULL.name())) + .numberOfShards(PRIMARIES) + .numberOfReplicas(REPLICAS) + .build(); + RoutingPool indexPool = getIndexPool(indexMetadata); + assertEquals(LOCAL_ONLY, indexPool); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java new file mode 100644 index 0000000000000..aba6fe74e0634 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/TieringAllocationBaseTestCase.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.Settings; + +import static org.opensearch.index.IndexModule.INDEX_STORE_LOCALITY_SETTING; +import static org.opensearch.index.IndexModule.INDEX_TIERING_STATE; + +@SuppressForbidden(reason = "feature flag overrides") +public abstract class TieringAllocationBaseTestCase extends RemoteShardsBalancerBaseTestCase { + + public ClusterState updateIndexMetadataForTiering( + ClusterState clusterState, + int localIndices, + String tieringState, + String dataLocality + ) { + Metadata.Builder mb = Metadata.builder(clusterState.metadata()); + for (int i = 0; i < localIndices; i++) { + IndexMetadata indexMetadata = clusterState.metadata().index(getIndexName(i, false)); + Settings settings = indexMetadata.getSettings(); + mb.put( + IndexMetadata.builder(indexMetadata) + .settings( + Settings.builder() + .put(settings) + .put(settings) + .put(INDEX_TIERING_STATE.getKey(), tieringState) + .put(INDEX_STORE_LOCALITY_SETTING.getKey(), dataLocality) + ) + ); + } + Metadata metadata = mb.build(); + return ClusterState.builder(clusterState).metadata(metadata).build(); + } +} From 442a92845391befb9ec7b58025d01683924c0282 Mon Sep 17 00:00:00 2001 From: Ganesh Krishna Ramadurai Date: Tue, 3 Sep 2024 14:19:17 -0700 Subject: [PATCH 031/357] pluggable deciders for concurrent search (#15363) Signed-off-by: Ganesh Ramadurai Signed-off-by: Jay Deng Co-authored-by: Jay Deng --- CHANGELOG.md | 1 + .../AggregationsIntegrationIT.java | 13 +- .../search/aggregations/CombiIT.java | 13 +- .../search/aggregations/EquivalenceIT.java | 13 +- .../search/aggregations/MetadataIT.java | 13 +- .../search/aggregations/MissingValueIT.java | 13 +- .../aggregations/pipeline/AvgBucketIT.java | 13 +- .../aggregations/pipeline/BucketScriptIT.java | 13 +- .../pipeline/BucketSelectorIT.java | 13 +- .../aggregations/pipeline/BucketSortIT.java | 13 +- .../pipeline/DateDerivativeIT.java | 13 +- .../aggregations/pipeline/DerivativeIT.java | 13 +- .../pipeline/ExtendedStatsBucketIT.java | 13 +- .../aggregations/pipeline/MaxBucketIT.java | 13 +- .../aggregations/pipeline/MinBucketIT.java | 13 +- .../aggregations/pipeline/MovAvgIT.java | 13 +- .../pipeline/PercentilesBucketIT.java | 13 +- .../aggregations/pipeline/SerialDiffIT.java | 13 +- .../aggregations/pipeline/StatsBucketIT.java | 13 +- .../aggregations/pipeline/SumBucketIT.java | 13 +- .../settings/AbstractScopedSettings.java | 30 ++ .../common/settings/ClusterSettings.java | 3 +- .../common/settings/IndexScopedSettings.java | 3 +- .../org/opensearch/index/IndexSettings.java | 24 +- .../main/java/org/opensearch/node/Node.java | 10 +- .../org/opensearch/plugins/SearchPlugin.java | 11 + .../search/DefaultSearchContext.java | 119 ++++- .../org/opensearch/search/SearchModule.java | 23 + .../org/opensearch/search/SearchService.java | 39 +- .../deciders/ConcurrentSearchDecider.java | 50 +++ .../deciders/ConcurrentSearchDecision.java | 88 ++++ .../deciders/ConcurrentSearchVisitor.java | 46 ++ .../search/deciders/package-info.java | 14 + .../common/settings/ScopedSettingsTests.java | 106 +++++ .../common/settings/SettingsModuleTests.java | 11 +- .../search/DefaultSearchContextTests.java | 407 +++++++++++++++++- .../opensearch/search/SearchModuleTests.java | 68 +++ .../opensearch/search/SearchServiceTests.java | 218 +++++++++- .../snapshots/SnapshotResiliencyTests.java | 3 +- .../java/org/opensearch/node/MockNode.java | 7 +- .../opensearch/search/MockSearchService.java | 4 +- .../ParameterizedOpenSearchIntegTestCase.java | 7 +- 42 files changed, 1422 insertions(+), 117 deletions(-) create mode 100644 server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java create mode 100644 server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java create mode 100644 server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java create mode 100644 server/src/main/java/org/opensearch/search/deciders/package-info.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a76bba98cd902..a4e5dd012c2eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410)) - [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) +- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java index 4051bee3e4e5c..01f3b434bc4dc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java @@ -57,7 +57,10 @@ import java.util.Collection; import java.util.List; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.stats; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -81,8 +84,12 @@ public AggregationsIntegrationIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java index 1826dd69cd804..4ce74f8195da9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/CombiIT.java @@ -51,7 +51,10 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.missing; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -69,8 +72,12 @@ public CombiIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java index 302ec3116d187..2fcf09d6ebf50 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/EquivalenceIT.java @@ -72,7 +72,10 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.extendedStats; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -102,8 +105,12 @@ public EquivalenceIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java index b650855083eed..ca13b4de00156 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MetadataIT.java @@ -49,7 +49,10 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.maxBucket; @@ -65,8 +68,12 @@ public MetadataIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java index bdd16c7e74dc0..1310792e2f2e4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java @@ -51,7 +51,10 @@ import java.util.Arrays; import java.util.Collection; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; @@ -73,8 +76,12 @@ public MissingValueIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java index 48fd06bac285b..7e1cae8e5b628 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/AvgBucketIT.java @@ -53,7 +53,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -83,8 +86,12 @@ public AvgBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java index 1b22cf2018d96..204aaa764849f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketScriptIT.java @@ -63,7 +63,10 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -96,8 +99,12 @@ public BucketScriptIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java index 7dca1d0d79b1e..7677cfdac3e29 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSelectorIT.java @@ -62,7 +62,10 @@ import java.util.function.Function; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.bucketSelector; @@ -95,8 +98,12 @@ public BucketSelectorIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java index ffb607866935b..cd489e5b982e6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/BucketSortIT.java @@ -58,7 +58,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -91,8 +94,12 @@ public BucketSortIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java index 8c89c1232ebb3..fc4c76f513f1d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DateDerivativeIT.java @@ -63,7 +63,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.derivative; @@ -89,8 +92,12 @@ public DateDerivativeIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java index f8def40ec003a..6fabbb32a4d15 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java @@ -61,7 +61,10 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.filters; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; @@ -106,8 +109,12 @@ public DerivativeIT(Settings dynamicSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java index 1bd04cc13268f..06d46e80f9710 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/ExtendedStatsBucketIT.java @@ -56,7 +56,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -86,8 +89,12 @@ public ExtendedStatsBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java index ea6fcbd6a1560..a8bd9bc4eadcd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MaxBucketIT.java @@ -66,7 +66,10 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.termQuery; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.filter; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; @@ -97,8 +100,12 @@ public MaxBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java index 44d12436382f6..290583ce49a5e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MinBucketIT.java @@ -53,7 +53,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -83,8 +86,12 @@ public MinBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java index d35b80b7918fe..e481e48cf6188 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java @@ -61,7 +61,10 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -139,8 +142,12 @@ public MovAvgIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java index 29cb334bfcd00..592a151781048 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/PercentilesBucketIT.java @@ -56,7 +56,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -86,8 +89,12 @@ public PercentilesBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java index 507bff51f0e39..2a412bc0f7720 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SerialDiffIT.java @@ -54,7 +54,10 @@ import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.max; @@ -104,8 +107,12 @@ public SerialDiffIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java index fbaf799871c8a..4e52e6e706324 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/StatsBucketIT.java @@ -53,7 +53,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -82,8 +85,12 @@ public StatsBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java index a5967124ff921..55652877d3f4b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/SumBucketIT.java @@ -53,7 +53,10 @@ import java.util.List; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.sum; import static org.opensearch.search.aggregations.AggregationBuilders.terms; @@ -83,8 +86,12 @@ public SumBucketIT(Settings staticSettings) { @ParametersFactory public static Collection parameters() { return Arrays.asList( - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() }, - new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() } + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_ALL).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() }, + new Object[] { + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_NONE).build() } ); } diff --git a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java index 117ed66fcb451..7655135b06d6c 100644 --- a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java @@ -790,6 +790,36 @@ public T get(Setting setting) { return setting.get(this.lastSettingsApplied, settings); } + /** + * Returns the value for the given setting if it is explicitly set, + * otherwise will return null instead of default value + **/ + public T getOrNull(Setting setting) { + if (setting.getProperties().contains(scope) == false) { + throw new SettingsException( + "settings scope doesn't match the setting scope [" + this.scope + "] not in [" + setting.getProperties() + "]" + ); + } + if (get(setting.getKey()) == null) { + throw new SettingsException("setting " + setting.getKey() + " has not been registered"); + } + if (setting.exists(lastSettingsApplied)) { + return setting.get(lastSettingsApplied); + } + if (setting.exists(settings)) { + return setting.get(settings); + } + if (setting.fallbackSetting != null) { + if (setting.fallbackSetting.exists(lastSettingsApplied)) { + return setting.fallbackSetting.get(lastSettingsApplied); + } + if (setting.fallbackSetting.exists(settings)) { + return setting.fallbackSetting.get(settings); + } + } + return null; + } + /** * Updates a target settings builder with new, updated or deleted settings from a given settings builder. *

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 8e036209daff0..c39040b503701 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -753,8 +753,9 @@ public void apply(Settings value, Settings current, Settings previous) { IoBasedAdmissionControllerSettings.INDEXING_IO_USAGE_LIMIT, // Concurrent segment search settings - SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING, + SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING, // deprecated SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING, + SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE, RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 00c73de1c1780..8d56a942c5d6e 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -238,7 +238,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING, // Settings for concurrent segment search - IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, + IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, // deprecated + IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MODE, IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MAX_SLICE_COUNT, IndexSettings.ALLOW_DERIVED_FIELDS, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 3e58b9df63378..554e99764c1a1 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -78,6 +78,9 @@ import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_DEFAULT_VALUE; /** @@ -699,7 +702,26 @@ public static IndexMergePolicy fromString(String text) { "index.search.concurrent_segment_search.enabled", false, Property.IndexScope, - Property.Dynamic + Property.Dynamic, + Property.Deprecated + ); + + public static final Setting INDEX_CONCURRENT_SEGMENT_SEARCH_MODE = Setting.simpleString( + "index.search.concurrent_segment_search.mode", + CONCURRENT_SEGMENT_SEARCH_MODE_NONE, + value -> { + switch (value) { + case CONCURRENT_SEGMENT_SEARCH_MODE_ALL: + case CONCURRENT_SEGMENT_SEARCH_MODE_NONE: + case CONCURRENT_SEGMENT_SEARCH_MODE_AUTO: + // valid setting + break; + default: + throw new IllegalArgumentException("Setting value must be one of [all, none, auto]"); + } + }, + Property.Dynamic, + Property.IndexScope ); public static final Setting INDEX_CONCURRENT_SEGMENT_SEARCH_MAX_SLICE_COUNT = Setting.intSetting( diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 563d184ed26a6..c55c2ab683b11 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -235,6 +235,7 @@ import org.opensearch.search.aggregations.support.AggregationUsageService; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.query.QueryPhase; @@ -1339,7 +1340,8 @@ protected Node( responseCollectorService, circuitBreakerService, searchModule.getIndexSearcherExecutor(threadPool), - taskResourceTrackingService + taskResourceTrackingService, + searchModule.getConcurrentSearchDeciders() ); final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) @@ -1997,7 +1999,8 @@ protected SearchService newSearchService( ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, - TaskResourceTrackingService taskResourceTrackingService + TaskResourceTrackingService taskResourceTrackingService, + Collection concurrentSearchDecidersList ) { return new SearchService( clusterService, @@ -2010,7 +2013,8 @@ protected SearchService newSearchService( responseCollectorService, circuitBreakerService, indexSearcherExecutor, - taskResourceTrackingService + taskResourceTrackingService, + concurrentSearchDecidersList ); } diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index 40b4f97cd1897..498da4042fa33 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.opensearch.common.CheckedFunction; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.lucene.search.function.ScoreFunction; import org.opensearch.common.settings.Settings; import org.opensearch.core.ParseField; @@ -64,6 +65,7 @@ import org.opensearch.search.aggregations.pipeline.MovAvgPipelineAggregator; import org.opensearch.search.aggregations.pipeline.PipelineAggregator; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.highlight.Highlighter; import org.opensearch.search.query.QueryPhaseSearcher; @@ -138,6 +140,15 @@ default Map getHighlighters() { return emptyMap(); } + /** + * Allows plugins to register custom decider for concurrent search + * @return A {@link ConcurrentSearchDecider} + */ + @ExperimentalApi + default ConcurrentSearchDecider getConcurrentSearchDecider() { + return null; + } + /** * The new {@link Suggester}s defined by this plugin. */ diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 1401c6b89d38a..fa762f9183d2a 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -32,6 +32,8 @@ package org.opensearch.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -70,6 +72,9 @@ import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.collapse.CollapseContext; +import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchDecision; +import org.opensearch.search.deciders.ConcurrentSearchVisitor; import org.opensearch.search.dfs.DfsSearchResult; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.fetch.FetchSearchResult; @@ -98,16 +103,23 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.stream.Collectors; import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS; /** @@ -117,11 +129,14 @@ */ final class DefaultSearchContext extends SearchContext { + private static final Logger logger = LogManager.getLogger(DefaultSearchContext.class); + private final ReaderContext readerContext; private final Engine.Searcher engineSearcher; private final ShardSearchRequest request; private final SearchShardTarget shardTarget; private final LongSupplier relativeTimeSupplier; + private final Collection concurrentSearchDeciders; private SearchType searchType; private final BigArrays bigArrays; private final IndexShard indexShard; @@ -187,7 +202,7 @@ final class DefaultSearchContext extends SearchContext { private final QueryShardContext queryShardContext; private final FetchPhase fetchPhase; private final Function requestToAggReduceContextBuilder; - private final boolean concurrentSearchSettingsEnabled; + private final String concurrentSearchMode; private final SetOnce requestShouldUseConcurrentSearch = new SetOnce<>(); private final int maxAggRewriteFilters; private final int cardinalityAggregationPruningThreshold; @@ -205,7 +220,8 @@ final class DefaultSearchContext extends SearchContext { Version minNodeVersion, boolean validate, Executor executor, - Function requestToAggReduceContextBuilder + Function requestToAggReduceContextBuilder, + Collection concurrentSearchDeciders ) throws IOException { this.readerContext = readerContext; this.request = request; @@ -221,14 +237,15 @@ final class DefaultSearchContext extends SearchContext { this.indexShard = readerContext.indexShard(); this.clusterService = clusterService; this.engineSearcher = readerContext.acquireSearcher("search"); - this.concurrentSearchSettingsEnabled = evaluateConcurrentSegmentSearchSettings(executor); + this.concurrentSearchMode = evaluateConcurrentSearchMode(executor); this.searcher = new ContextIndexSearcher( engineSearcher.getIndexReader(), engineSearcher.getSimilarity(), engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation, - concurrentSearchSettingsEnabled ? executor : null, + concurrentSearchMode.equals(CONCURRENT_SEGMENT_SEARCH_MODE_AUTO) + || concurrentSearchMode.equals(CONCURRENT_SEGMENT_SEARCH_MODE_ALL) ? executor : null, this ); this.relativeTimeSupplier = relativeTimeSupplier; @@ -247,6 +264,7 @@ final class DefaultSearchContext extends SearchContext { this.maxAggRewriteFilters = evaluateFilterRewriteSetting(); this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold(); + this.concurrentSearchDeciders = concurrentSearchDeciders; } @Override @@ -902,11 +920,68 @@ public Profilers getProfilers() { @Override public boolean shouldUseConcurrentSearch() { assert requestShouldUseConcurrentSearch.get() != null : "requestShouldUseConcurrentSearch must be set"; - return concurrentSearchSettingsEnabled && Boolean.TRUE.equals(requestShouldUseConcurrentSearch.get()); + assert concurrentSearchMode != null : "concurrentSearchMode must be set"; + return (concurrentSearchMode.equals(CONCURRENT_SEGMENT_SEARCH_MODE_AUTO) + || concurrentSearchMode.equals(CONCURRENT_SEGMENT_SEARCH_MODE_ALL)) + && Boolean.TRUE.equals(requestShouldUseConcurrentSearch.get()); + } + + private boolean evaluateAutoMode() { + + // filter out deciders that want to opt-out of decision-making + final Set filteredDeciders = concurrentSearchDeciders.stream() + .filter(concurrentSearchDecider -> concurrentSearchDecider.canEvaluateForIndex(indexService.getIndexSettings())) + .collect(Collectors.toSet()); + // evaluate based on concurrent search query visitor + if (filteredDeciders.size() > 0) { + ConcurrentSearchVisitor concurrentSearchVisitor = new ConcurrentSearchVisitor( + filteredDeciders, + indexService.getIndexSettings() + ); + if (request().source() != null && request().source().query() != null) { + QueryBuilder queryBuilder = request().source().query(); + queryBuilder.visit(concurrentSearchVisitor); + } + } + + final List decisions = new ArrayList<>(); + for (ConcurrentSearchDecider decider : filteredDeciders) { + ConcurrentSearchDecision decision = decider.getConcurrentSearchDecision(); + if (decision != null) { + if (logger.isDebugEnabled()) { + logger.debug("concurrent search decision from plugin decider [{}]", decision.toString()); + } + decisions.add(decision); + } + } + + final ConcurrentSearchDecision pluginDecision = ConcurrentSearchDecision.getCompositeDecision(decisions); + if (pluginDecision.getDecisionStatus().equals(ConcurrentSearchDecision.DecisionStatus.NO_OP)) { + // plugins don't have preference, decide based on whether request has aggregations or not. + if (aggregations() != null) { + if (logger.isDebugEnabled()) { + logger.debug("request has supported aggregations, using concurrent search"); + } + return true; + + } else { + if (logger.isDebugEnabled()) { + logger.debug("request does not have aggregations, not using concurrent search"); + } + return false; + } + + } else { + if (logger.isDebugEnabled()) { + logger.debug("concurrent search decision from plugins [{}]", pluginDecision.toString()); + } + return pluginDecision.getDecisionStatus() == ConcurrentSearchDecision.DecisionStatus.YES; + } + } /** - * Evaluate if parsed request supports concurrent segment search + * Evaluate if request should use concurrent search based on request and concurrent search deciders */ public void evaluateRequestShouldUseConcurrentSearch() { if (sort != null && sort.isSortOnTimeSeriesField()) { @@ -917,6 +992,8 @@ && aggregations().factories() != null requestShouldUseConcurrentSearch.set(false); } else if (terminateAfter != DEFAULT_TERMINATE_AFTER) { requestShouldUseConcurrentSearch.set(false); + } else if (concurrentSearchMode.equals(CONCURRENT_SEGMENT_SEARCH_MODE_AUTO)) { + requestShouldUseConcurrentSearch.set(evaluateAutoMode()); } else { requestShouldUseConcurrentSearch.set(true); } @@ -964,26 +1041,40 @@ public BucketCollectorProcessor bucketCollectorProcessor() { } /** - * Evaluate based on cluster and index settings if concurrent segment search should be used for this request context - * @return true: use concurrent search - * false: otherwise + * Evaluate the concurrentSearchMode based on cluster and index settings if concurrent segment search + * should be used for this request context + * If the cluster.search.concurrent_segment_search.mode setting + * is not explicitly set, the evaluation falls back to the + * cluster.search.concurrent_segment_search.enabled boolean setting + * which will evaluate to true or false. This is then evaluated to "all" or "none" respectively + * @return one of "none", "auto", "all" */ - private boolean evaluateConcurrentSegmentSearchSettings(Executor concurrentSearchExecutor) { + private String evaluateConcurrentSearchMode(Executor concurrentSearchExecutor) { // Do not use concurrent segment search for system indices or throttled requests. See: // https://github.com/opensearch-project/OpenSearch/issues/12951 if (indexShard.isSystem() || indexShard.indexSettings().isSearchThrottled()) { - return false; + return CONCURRENT_SEGMENT_SEARCH_MODE_NONE; } + if ((clusterService != null) && concurrentSearchExecutor != null) { + String concurrentSearchMode = indexService.getIndexSettings() + .getSettings() + .get( + IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), + clusterService.getClusterSettings().getOrNull(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE) + ); + if (concurrentSearchMode != null) { + return concurrentSearchMode; + } - if ((clusterService != null) && (concurrentSearchExecutor != null)) { + // mode setting not set, fallback to concurrent_segment_search.enabled setting return indexService.getIndexSettings() .getSettings() .getAsBoolean( IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), clusterService.getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) - ); + ) ? CONCURRENT_SEGMENT_SEARCH_MODE_ALL : CONCURRENT_SEGMENT_SEARCH_MODE_NONE; } - return false; + return CONCURRENT_SEGMENT_SEARCH_MODE_NONE; } @Override diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index b463458847a88..c51004a1ea95e 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -239,6 +239,7 @@ import org.opensearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; import org.opensearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.ExplainPhase; @@ -282,6 +283,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -316,6 +318,8 @@ public class SearchModule { private final QueryPhaseSearcher queryPhaseSearcher; private final SearchPlugin.ExecutorServiceProvider indexSearcherExecutorProvider; + private final Collection concurrentSearchDeciders; + /** * Constructs a new SearchModule object *

@@ -344,6 +348,25 @@ public SearchModule(Settings settings, List plugins) { queryPhaseSearcher = registerQueryPhaseSearcher(plugins); indexSearcherExecutorProvider = registerIndexSearcherExecutorProvider(plugins); namedWriteables.addAll(SortValue.namedWriteables()); + concurrentSearchDeciders = registerConcurrentSearchDeciders(plugins); + } + + private Collection registerConcurrentSearchDeciders(List plugins) { + List concurrentSearchDeciders = new ArrayList<>(); + for (SearchPlugin plugin : plugins) { + ConcurrentSearchDecider decider = plugin.getConcurrentSearchDecider(); + if (decider != null) { + concurrentSearchDeciders.add(decider); + } + } + return concurrentSearchDeciders; + } + + /** + * Returns the concurrent search deciders that the plugins have registered + */ + public Collection getConcurrentSearchDeciders() { + return concurrentSearchDeciders; } public List getNamedWriteables() { diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index a53a7198c366f..39c0c19a978e5 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -104,6 +104,7 @@ import org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.collapse.CollapseContext; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.dfs.DfsPhase; import org.opensearch.search.dfs.DfsSearchResult; import org.opensearch.search.fetch.FetchPhase; @@ -146,6 +147,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -256,6 +258,34 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv "search.concurrent_segment_search.enabled", false, Property.Dynamic, + Property.NodeScope, + Property.Deprecated + ); + + // Allow concurrent segment search for all requests + public static final String CONCURRENT_SEGMENT_SEARCH_MODE_ALL = "all"; + + // Disallow concurrent search for all requests + public static final String CONCURRENT_SEGMENT_SEARCH_MODE_NONE = "none"; + + // Make decision for concurrent search based on concurrent search deciders + public static final String CONCURRENT_SEGMENT_SEARCH_MODE_AUTO = "auto"; + + public static final Setting CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE = Setting.simpleString( + "search.concurrent_segment_search.mode", + CONCURRENT_SEGMENT_SEARCH_MODE_NONE, + value -> { + switch (value) { + case CONCURRENT_SEGMENT_SEARCH_MODE_ALL: + case CONCURRENT_SEGMENT_SEARCH_MODE_NONE: + case CONCURRENT_SEGMENT_SEARCH_MODE_AUTO: + // valid setting + break; + default: + throw new IllegalArgumentException("Setting value must be one of [all, none, auto]"); + } + }, + Property.Dynamic, Property.NodeScope ); @@ -327,6 +357,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final QueryPhase queryPhase; private final FetchPhase fetchPhase; + private final Collection concurrentSearchDeciders; private volatile long defaultKeepAlive; @@ -371,7 +402,8 @@ public SearchService( ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, - TaskResourceTrackingService taskResourceTrackingService + TaskResourceTrackingService taskResourceTrackingService, + Collection concurrentSearchDeciders ) { Settings settings = clusterService.getSettings(); this.threadPool = threadPool; @@ -426,6 +458,8 @@ public SearchService( allowDerivedField = CLUSTER_ALLOW_DERIVED_FIELD_SETTING.get(settings); clusterService.getClusterSettings().addSettingsUpdateConsumer(CLUSTER_ALLOW_DERIVED_FIELD_SETTING, this::setAllowDerivedField); + + this.concurrentSearchDeciders = concurrentSearchDeciders; } private void validateKeepAlives(TimeValue defaultKeepAlive, TimeValue maxKeepAlive) { @@ -1125,7 +1159,8 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear clusterService.state().nodes().getMinNodeVersion(), validate, indexSearcherExecutor, - this::aggReduceContextBuilder + this::aggReduceContextBuilder, + concurrentSearchDeciders ); // we clone the query shard context here just for rewriting otherwise we // might end up with incorrect state since we are using now() or script services diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java new file mode 100644 index 0000000000000..9c588bb45b4ec --- /dev/null +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.deciders; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.query.QueryBuilder; + +/** + * {@link ConcurrentSearchDecider} allows pluggable way to evaluate if a query in the search request + * can use concurrent segment search using the passed in queryBuilders from query tree and index settings + * on a per shard request basis. + * Implementations can also opt out of the evaluation process for certain indices based on the index settings. + * For all the deciders which can evaluate query tree for an index, its evaluateForQuery method + * will be called for each node in the query tree. After traversing of the query tree is completed, the final + * decision from the deciders will be obtained using {@link ConcurrentSearchDecider#getConcurrentSearchDecision} + */ +@ExperimentalApi +public abstract class ConcurrentSearchDecider { + + /** + * Evaluate for the passed in queryBuilder node in the query tree of the search request + * if concurrent segment search can be used. + * This method will be called for each of the query builder node in the query tree of the request. + */ + public abstract void evaluateForQuery(QueryBuilder queryBuilder, IndexSettings indexSettings); + + /** + * Provides a way for deciders to opt out of decision-making process for certain requests based on + * index settings. + * Return true if interested in decision making for index, + * false, otherwise + */ + public abstract boolean canEvaluateForIndex(IndexSettings indexSettings); + + /** + * Provide the final decision for concurrent search based on all evaluations + * Plugins may need to maintain internal state of evaluations to provide a final decision + * If decision is null, then it is ignored + * @return ConcurrentSearchDecision + */ + public abstract ConcurrentSearchDecision getConcurrentSearchDecision(); + +} diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java new file mode 100644 index 0000000000000..2a30413eff9c8 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.deciders; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Collection; + +/** + * This Class defines the decisions that a {@link ConcurrentSearchDecider#getConcurrentSearchDecision} can return. + * + */ +@ExperimentalApi +public class ConcurrentSearchDecision { + + final private DecisionStatus decisionStatus; + final private String decisionReason; + + public ConcurrentSearchDecision(DecisionStatus decisionStatus, String decisionReason) { + this.decisionStatus = decisionStatus; + this.decisionReason = decisionReason; + } + + public DecisionStatus getDecisionStatus() { + return decisionStatus; + } + + public String getDecisionReason() { + return decisionReason; + } + + /** + * This enum contains the decision status for concurrent search. + */ + @ExperimentalApi + public enum DecisionStatus { + YES(0), // use concurrent search + NO(1), // don't use concurrent search + NO_OP(2); // no preference + + private final int id; + + DecisionStatus(int id) { + this.id = id; + } + } + + @Override + public String toString() { + return "ConcurrentSearchDecision{" + "decisionStatus=" + decisionStatus + ", decisionReason='" + decisionReason + '\'' + '}'; + } + + /** + * Combine a collection of {@link ConcurrentSearchDecision} to return final {@link ConcurrentSearchDecision} + * The decisions are combined as: + * NO_OP AND NO_OP results in NO_OP + * NO_OP AND YES results in YES + * NO_OP AND NO results in NO + */ + public static ConcurrentSearchDecision getCompositeDecision(Collection allDecisions) { + + DecisionStatus finalDecisionStatus = DecisionStatus.NO_OP; + for (ConcurrentSearchDecision decision : allDecisions) { + switch (decision.decisionStatus) { + case YES: + finalDecisionStatus = DecisionStatus.YES; + break; + case NO: + finalDecisionStatus = DecisionStatus.NO; + return new ConcurrentSearchDecision( + finalDecisionStatus, + "composite decision evaluated to false due to " + decision.decisionReason + ); + case NO_OP: + // NOOP doesn't change the final decision + break; + } + } + return new ConcurrentSearchDecision(finalDecisionStatus, "composite decision result"); + } + +} diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java new file mode 100644 index 0000000000000..12ba1b2a9cc5f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.deciders; + +import org.apache.lucene.search.BooleanClause; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilderVisitor; + +import java.util.Objects; +import java.util.Set; + +/** + * Class to traverse the QueryBuilder tree and invoke the + * {@link ConcurrentSearchDecider#evaluateForQuery} at each node of the query tree + */ +@ExperimentalApi +public class ConcurrentSearchVisitor implements QueryBuilderVisitor { + + private final Set deciders; + private final IndexSettings indexSettings; + + public ConcurrentSearchVisitor(Set concurrentSearchVisitorDeciders, IndexSettings idxSettings) { + Objects.requireNonNull(concurrentSearchVisitorDeciders, "Concurrent search deciders cannot be null"); + deciders = concurrentSearchVisitorDeciders; + indexSettings = idxSettings; + } + + @Override + public void accept(QueryBuilder qb) { + // for each of the deciders, invoke evaluateForQuery using the current query builder and index settings. + deciders.forEach(concurrentSearchDecider -> { concurrentSearchDecider.evaluateForQuery(qb, indexSettings); }); + } + + @Override + public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) { + return this; + } +} diff --git a/server/src/main/java/org/opensearch/search/deciders/package-info.java b/server/src/main/java/org/opensearch/search/deciders/package-info.java new file mode 100644 index 0000000000000..4ef26f7be9cfc --- /dev/null +++ b/server/src/main/java/org/opensearch/search/deciders/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains classes assist in deciding + * whether to run a search request using concurrent search or not. + */ + +package org.opensearch.search.deciders; diff --git a/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java index 0c5cece4249ef..7780481c9deff 100644 --- a/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java @@ -61,6 +61,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.startsWith; @@ -1478,4 +1481,107 @@ public void testAddSettingsUpdateConsumer() { if (i == 42) throw new AssertionError("empty key"); })); } + + public void testGetOrNullWhenSettingIsNotSet() { + Setting testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); + Setting testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope); + AbstractScopedSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(testSetting, testSetting2)) + ); + clusterSettings.registerSetting(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE); + clusterSettings.registerSetting(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING); + assertNull(clusterSettings.getOrNull(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)); + + } + + public void testGetOrNullWhenSettingIsSet() { + Setting testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); + Setting testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope); + AbstractScopedSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(testSetting, testSetting2)) + ); + clusterSettings.registerSetting(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE); + clusterSettings.applySettings( + Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO).build() + ); + assertEquals(clusterSettings.getOrNull(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE), "auto"); + + } + + public void testGetOrNullWhenSettingIsSetInNodeSettings() { + Setting testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); + Setting testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope); + Settings concurrentSearchModeSetting = Settings.builder() + .put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), CONCURRENT_SEGMENT_SEARCH_MODE_AUTO) + .build(); + AbstractScopedSettings clusterSettings = new ClusterSettings( + concurrentSearchModeSetting, + new HashSet<>(Arrays.asList(testSetting, testSetting2, CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE)) + ); + assertEquals(clusterSettings.getOrNull(CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE), "auto"); + + } + + public void testGetOrNullWhenSettingScopeDoesntMatch() { + Setting testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); + Setting testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope); + AbstractScopedSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(testSetting, testSetting2)) + ); + Setting failedSetting = Setting.intSetting("foo.bar.scope.fail", 1, Property.Dynamic, Property.IndexScope); + clusterSettings.registerSetting(failedSetting); + try { + clusterSettings.getOrNull(failedSetting); + fail("setting scope doesn't match"); + } catch (SettingsException ex) { + assertEquals("settings scope doesn't match the setting scope [NodeScope] not in [[Dynamic, IndexScope]]", ex.getMessage()); + } + + } + + public void testGetOrNullWhenSettingIsNotRegistered() { + Setting testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); + Setting testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope); + AbstractScopedSettings clusterSettings = new ClusterSettings( + Settings.EMPTY, + new HashSet<>(Arrays.asList(testSetting, testSetting2)) + ); + Setting failedSetting = Setting.intSetting("foo.bar.register.fail", 1, Property.Dynamic, Property.NodeScope); + + try { + clusterSettings.getOrNull(failedSetting); + fail("setting is not registered"); + } catch (SettingsException ex) { + assertEquals("setting foo.bar.register.fail has not been registered", ex.getMessage()); + } + + } + + public void testGetOrNullWhenSettingIsRegisteredWithFallback() { + Setting fallbackSetting = Setting.intSetting("foo.bar", 10, Property.Dynamic, Property.NodeScope); + AbstractScopedSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fallbackSetting))); + clusterSettings.registerSetting(fallbackSetting); + clusterSettings.applySettings(Settings.builder().put(fallbackSetting.getKey(), 100).build()); + Setting settingWithFallback = Setting.intSetting("foo.fallback", fallbackSetting, 1, Property.Dynamic, Property.NodeScope); + clusterSettings.registerSetting(settingWithFallback); + + assertEquals(clusterSettings.getOrNull(settingWithFallback), Integer.valueOf(100)); + + } + + public void testGetOrNullWhenSettingIsRegisteredNodeSettingFallback() { + Setting fallbackSetting = Setting.intSetting("foo.bar", 10, Property.Dynamic, Property.NodeScope); + Settings settings = Settings.builder().put(fallbackSetting.getKey(), 100).build(); + AbstractScopedSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(Arrays.asList(fallbackSetting))); + + Setting settingWithFallback = Setting.intSetting("foo.fallback", fallbackSetting, 1, Property.Dynamic, Property.NodeScope); + clusterSettings.registerSetting(settingWithFallback); + + assertEquals(clusterSettings.getOrNull(settingWithFallback), Integer.valueOf(100)); + + } + } diff --git a/server/src/test/java/org/opensearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/opensearch/common/settings/SettingsModuleTests.java index 66c9801d16598..d504c3af90679 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingsModuleTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingsModuleTests.java @@ -290,20 +290,27 @@ public void testConcurrentSegmentSearchClusterSettings() { Settings settings = Settings.builder().put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), settingValue).build(); SettingsModule settingsModule = new SettingsModule(settings); assertEquals(settingValue, SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.get(settingsModule.getSettings())); + assertSettingDeprecationsAndWarnings(new Setting[] { SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING }); } public void testConcurrentSegmentSearchIndexSettings() { Settings.Builder target = Settings.builder().put(Settings.EMPTY); Settings.Builder update = Settings.builder(); - + boolean settingValue = randomBoolean(); SettingsModule module = new SettingsModule(Settings.EMPTY); IndexScopedSettings indexScopedSettings = module.getIndexScopedSettings(); indexScopedSettings.updateDynamicSettings( - Settings.builder().put(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build(), + Settings.builder().put(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), settingValue).build(), target, update, "node" ); + // apply the setting update + module.getIndexScopedSettings() + .applySettings(Settings.builder().put(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), settingValue).build()); + // assert value + assertEquals(settingValue, module.getIndexScopedSettings().get(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING)); + assertSettingDeprecationsAndWarnings(new Setting[] { IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING }); } public void testMaxSliceCountClusterSettingsForConcurrentSearch() { diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index a1a808c9faa9b..491a0377ab32e 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -51,6 +51,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; @@ -67,9 +68,16 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.AbstractQueryBuilder; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ParsedQuery; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; +import org.opensearch.search.aggregations.AggregatorFactories; +import org.opensearch.search.aggregations.MultiBucketConsumerService; +import org.opensearch.search.aggregations.SearchContextAggregations; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchDecision; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.LegacyReaderContext; import org.opensearch.search.internal.PitReaderContext; @@ -84,8 +92,10 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -220,7 +230,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); contextWithoutScroll.from(300); contextWithoutScroll.close(); @@ -263,7 +274,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context1.from(300); exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false)); @@ -334,7 +346,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); SliceBuilder sliceBuilder = mock(SliceBuilder.class); @@ -374,7 +387,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery(); context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false); @@ -410,7 +424,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context4.sliceBuilder(new SliceBuilder(1, 2)).parsedQuery(parsedQuery).preProcess(false); Query query1 = context4.query(); @@ -441,7 +456,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); int numSlicesForPit = maxSlicesPerPit + randomIntBetween(1, 100); when(sliceBuilder.getMax()).thenReturn(numSlicesForPit); @@ -547,7 +563,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); assertThat(context.searcher().hasCancellations(), is(false)); context.searcher().addQueryCancellation(() -> {}); @@ -643,6 +660,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { final ClusterService clusterService = mock(ClusterService.class); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); clusterSettings.registerSetting(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING); + // clusterSettings.registerSetting(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE); clusterSettings.applySettings( Settings.builder().put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() ); @@ -651,7 +669,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { readerContext, shardSearchRequest, target, - null, + clusterService, bigArrays, null, null, @@ -660,7 +678,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); // Case1: if sort is on timestamp field, non-concurrent path is used @@ -685,7 +704,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context.sort( new SortAndFormats(new Sort(new SortField("test2", SortField.Type.INT)), new DocValueFormat[] { DocValueFormat.RAW }) @@ -712,7 +732,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context.evaluateRequestShouldUseConcurrentSearch(); if (executor == null) { @@ -744,7 +765,8 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context.evaluateRequestShouldUseConcurrentSearch(); assertFalse(context.shouldUseConcurrentSearch()); @@ -772,12 +794,371 @@ protected Engine.Searcher acquireSearcherInternal(String source) { Version.CURRENT, false, executor, - null + null, + Collections.emptyList() ); context.evaluateRequestShouldUseConcurrentSearch(); assertFalse(context.shouldUseConcurrentSearch()); assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + if (clusterService.getClusterSettings().get(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)) { + assertSettingDeprecationsAndWarnings(new Setting[] { SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING }); + } + + // shutdown the threadpool + threadPool.shutdown(); + } + } + + public void testSearchPathEvaluationWithConcurrentSearchModeAsAuto() throws Exception { + ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class); + when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); + ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1); + when(shardSearchRequest.shardId()).thenReturn(shardId); + + ThreadPool threadPool = new TestThreadPool(this.getClass().getName()); + IndexShard indexShard = mock(IndexShard.class); + QueryCachingPolicy queryCachingPolicy = mock(QueryCachingPolicy.class); + when(indexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy); + when(indexShard.getThreadPool()).thenReturn(threadPool); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2) + .build(); + + IndexService indexService = mock(IndexService.class); + QueryShardContext queryShardContext = mock(QueryShardContext.class); + when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn( + queryShardContext + ); + + IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + when(indexService.getIndexSettings()).thenReturn(indexSettings); + when(indexShard.indexSettings()).thenReturn(indexSettings); + + BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); + + IndexShard systemIndexShard = mock(IndexShard.class); + when(systemIndexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy); + when(systemIndexShard.getThreadPool()).thenReturn(threadPool); + when(systemIndexShard.isSystem()).thenReturn(true); + + IndexShard throttledIndexShard = mock(IndexShard.class); + when(throttledIndexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy); + when(throttledIndexShard.getThreadPool()).thenReturn(threadPool); + IndexSettings throttledIndexSettings = new IndexSettings( + indexMetadata, + Settings.builder().put(INDEX_SEARCH_THROTTLED.getKey(), true).build() + ); + when(throttledIndexShard.indexSettings()).thenReturn(throttledIndexSettings); + + try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + + final Supplier searcherSupplier = () -> new Engine.SearcherSupplier(Function.identity()) { + @Override + protected void doClose() {} + + @Override + protected Engine.Searcher acquireSearcherInternal(String source) { + try { + IndexReader reader = w.getReader(); + return new Engine.Searcher( + "test", + reader, + IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), + reader + ); + } catch (IOException exc) { + throw new AssertionError(exc); + } + } + }; + + SearchShardTarget target = new SearchShardTarget("node", shardId, null, OriginalIndices.NONE); + ReaderContext readerContext = new ReaderContext( + newContextId(), + indexService, + indexShard, + searcherSupplier.get(), + randomNonNegativeLong(), + false + ); + + final ClusterService clusterService = mock(ClusterService.class); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + clusterSettings.registerSetting(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING); + clusterSettings.registerSetting(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE); + clusterSettings.applySettings( + Settings.builder().put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), "auto").build() + ); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + DefaultSearchContext context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + Collections.emptyList() + ); + + // Case1: if there is no agg in the query, non-concurrent path is used + context.evaluateRequestShouldUseConcurrentSearch(); + assertFalse(context.shouldUseConcurrentSearch()); + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case2: if un supported agg present, non-concurrent path is used + SearchContextAggregations mockAggregations = mock(SearchContextAggregations.class); + when(mockAggregations.factories()).thenReturn(mock(AggregatorFactories.class)); + when(mockAggregations.factories().allFactoriesSupportConcurrentSearch()).thenReturn(false); + when(mockAggregations.multiBucketConsumer()).thenReturn(mock(MultiBucketConsumerService.MultiBucketConsumer.class)); + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + Collections.emptyList() + ); + + // add un-supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertFalse(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case3: if supported agg present, concurrent path is used + + // set agg operation to be supported + when(mockAggregations.factories().allFactoriesSupportConcurrentSearch()).thenReturn(true); + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + Collections.emptyList() + ); + // create a supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertTrue(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case4: multiple deciders are registered and all of them opt out of decision-making + // with supported agg query so concurrent path is used + + ConcurrentSearchDecider decider1 = mock(ConcurrentSearchDecider.class); + when(decider1.canEvaluateForIndex(any())).thenReturn(false); + ConcurrentSearchDecider decider2 = mock(ConcurrentSearchDecider.class); + when(decider2.canEvaluateForIndex(any())).thenReturn(false); + + Collection concurrentSearchDeciders = new ArrayList<>(); + concurrentSearchDeciders.add(decider1); + concurrentSearchDeciders.add(decider2); + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + concurrentSearchDeciders + ); + // create a supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertTrue(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case5: multiple deciders are registered and one of them returns ConcurrentSearchDecision.DecisionStatus.NO + // use non-concurrent path even if query contains supported agg + when(decider1.canEvaluateForIndex(any())).thenReturn(true); + when(decider1.getConcurrentSearchDecision()).thenReturn( + new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO, "disable concurrent search") + ); + when(decider2.canEvaluateForIndex(any())).thenReturn(false); + + concurrentSearchDeciders.clear(); + concurrentSearchDeciders.add(decider1); + concurrentSearchDeciders.add(decider2); + + // create a source so that query tree is parsed by visitor + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); + sourceBuilder.query(queryBuilder); + when(shardSearchRequest.source()).thenReturn(sourceBuilder); + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + concurrentSearchDeciders + ); + + // create a supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertFalse(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case6: multiple deciders are registered and first decider returns ConcurrentSearchDecision.DecisionStatus.YES + // while second decider returns ConcurrentSearchDecision.DecisionStatus.NO + // use non-concurrent path even if query contains supported agg + when(decider1.canEvaluateForIndex(any())).thenReturn(true); + when(decider1.getConcurrentSearchDecision()).thenReturn( + new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.YES, "enable concurrent search") + ); + when(decider2.canEvaluateForIndex(any())).thenReturn(true); + when(decider2.getConcurrentSearchDecision()).thenReturn( + new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO, "disable concurrent search") + ); + + concurrentSearchDeciders.clear(); + concurrentSearchDeciders.add(decider1); + concurrentSearchDeciders.add(decider2); + + // create a source so that query tree is parsed by visitor + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + concurrentSearchDeciders + ); + + // create a supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertFalse(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + + // Case7: multiple deciders are registered and all return ConcurrentSearchDecision.DecisionStatus.NO_OP + // but un-supported agg query is present, use non-concurrent path + when(decider1.canEvaluateForIndex(any())).thenReturn(true); + when(decider1.getConcurrentSearchDecision()).thenReturn( + new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO_OP, "noop") + ); + when(decider2.canEvaluateForIndex(any())).thenReturn(true); + when(decider2.getConcurrentSearchDecision()).thenReturn( + new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO_OP, "noop") + ); + + when(mockAggregations.factories().allFactoriesSupportConcurrentSearch()).thenReturn(false); + + concurrentSearchDeciders.clear(); + concurrentSearchDeciders.add(decider1); + concurrentSearchDeciders.add(decider2); + + // create a source so that query tree is parsed by visitor + + context = new DefaultSearchContext( + readerContext, + shardSearchRequest, + target, + clusterService, + bigArrays, + null, + null, + null, + false, + Version.CURRENT, + false, + executor, + null, + concurrentSearchDeciders + ); + + // create a supported agg operation + context.aggregations(mockAggregations); + context.evaluateRequestShouldUseConcurrentSearch(); + if (executor == null) { + assertFalse(context.shouldUseConcurrentSearch()); + } else { + assertFalse(context.shouldUseConcurrentSearch()); + } + assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch); + // shutdown the threadpool threadPool.shutdown(); } diff --git a/server/src/test/java/org/opensearch/search/SearchModuleTests.java b/server/src/test/java/org/opensearch/search/SearchModuleTests.java index 01b8d6d8cdd72..71bbe7c718f47 100644 --- a/server/src/test/java/org/opensearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/opensearch/search/SearchModuleTests.java @@ -69,6 +69,7 @@ import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.aggregations.support.ValuesSourceType; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.ExplainPhase; import org.opensearch.search.fetch.subphase.highlight.CustomHighlighter; @@ -500,6 +501,73 @@ public Optional getIndexSearcherExecutorProvider() { expectThrows(IllegalStateException.class, () -> new SearchModule(Settings.EMPTY, searchPlugins)); } + public void testRegisterConcurrentSearchDecidersNoExternalPlugins() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); + } + + public void testRegisterConcurrentSearchDecidersExternalPluginsWithNoDeciders() { + SearchPlugin plugin1 = new SearchPlugin() { + @Override + public Optional getIndexSearcherExecutorProvider() { + return Optional.of(mock(ExecutorServiceProvider.class)); + } + }; + SearchPlugin plugin2 = new SearchPlugin() { + }; + + List searchPlugins = new ArrayList<>(); + searchPlugins.add(plugin1); + searchPlugins.add(plugin2); + SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); + + assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); + } + + public void testRegisterConcurrentSearchDecidersExternalPluginsWithDeciders() { + SearchPlugin pluginDecider1 = new SearchPlugin() { + @Override + public Optional getIndexSearcherExecutorProvider() { + return Optional.of(mock(ExecutorServiceProvider.class)); + } + + @Override + public ConcurrentSearchDecider getConcurrentSearchDecider() { + return mock(ConcurrentSearchDecider.class); + } + }; + + SearchPlugin pluginDecider2 = new SearchPlugin() { + @Override + public ConcurrentSearchDecider getConcurrentSearchDecider() { + return mock(ConcurrentSearchDecider.class); + } + }; + + List searchPlugins = new ArrayList<>(); + searchPlugins.add(pluginDecider1); + searchPlugins.add(pluginDecider2); + + SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); + assertEquals(searchModule.getConcurrentSearchDeciders().size(), 2); + } + + public void testRegisterConcurrentSearchDecidersPluginWithNullDecider() { + SearchPlugin pluginWithNullDecider = new SearchPlugin() { + @Override + public ConcurrentSearchDecider getConcurrentSearchDecider() { + return null; + } + }; + + List searchPlugins = new ArrayList<>(); + searchPlugins.add(pluginWithNullDecider); + SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); + // null decider is filtered out, so 0 deciders + assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); + + } + private static final String[] NON_DEPRECATED_QUERIES = new String[] { "bool", "boosting", diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index 9fb4db96cbf7c..41c77866f8167 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -56,6 +56,7 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; @@ -93,8 +94,10 @@ import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.MultiBucketConsumerService; +import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; @@ -141,6 +144,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SearchServiceTests extends OpenSearchSingleNodeTestCase { @@ -1314,17 +1319,40 @@ public void testCreateSearchContext() throws IOException { * index and cluster settings. */ public void testConcurrentSegmentSearchSearchContext() throws IOException { - Boolean[][] scenarios = { - // cluster setting, index setting, concurrent search enabled? - { null, null, false }, - { null, false, false }, - { null, true, true }, - { true, null, true }, - { true, false, false }, - { true, true, true }, - { false, null, false }, - { false, false, false }, - { false, true, true } }; + Object[][] scenarios = { + // cluster setting, index setting, cluster mode setting, concurrent search enabled?, concurrent search executor null? + { null, null, null, false, true }, + { null, false, null, false, true }, + { null, true, null, true, false }, + { true, null, null, true, false }, + { true, false, null, false, true }, + { true, true, null, true, false }, + { false, null, null, false, true }, + { false, false, null, false, true }, + { false, true, null, true, false }, + + // Adding cases with mode set to "none" + { null, null, "none", false, true }, + { true, true, "none", false, true }, + { false, false, "none", false, true }, + { true, false, "none", false, true }, + { false, true, "none", false, true }, + + // Adding cases with mode set to "all" + { null, null, "all", true, false }, + { true, true, "all", true, false }, + { false, false, "all", true, false }, + { true, false, "all", true, false }, + { false, true, "all", true, false }, + + // Adding cases with mode set to "auto" + // auto mode concurrent search is false since request has no aggregation + // however concurrentSearchExecutor will not be null + { null, null, "auto", false, false }, + { true, true, "auto", false, false }, + { false, false, "auto", false, false }, + { true, false, "auto", false, false }, + { false, true, "auto", false, false } }; String index = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); IndexService indexService = createIndex(index); @@ -1346,10 +1374,12 @@ public void testConcurrentSegmentSearchSearchContext() throws IOException { Strings.EMPTY_ARRAY ); - for (Boolean[] scenario : scenarios) { - Boolean clusterSetting = scenario[0]; - Boolean indexSetting = scenario[1]; - Boolean concurrentSearchEnabled = scenario[2]; + for (Object[] scenario : scenarios) { + Boolean clusterSetting = (Boolean) scenario[0]; + Boolean indexSetting = (Boolean) scenario[1]; + String mode = (String) scenario[2]; + Boolean concurrentSearchEnabled = (Boolean) scenario[3]; + Boolean concurrentSearchExecutorNull = (Boolean) scenario[4]; if (clusterSetting == null) { client().admin() @@ -1381,6 +1411,21 @@ public void testConcurrentSegmentSearchSearchContext() throws IOException { .get(); } + // update mode + if (mode == null) { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey())) + .get(); + } else { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), mode)) + .get(); + } + try (DefaultSearchContext searchContext = service.createSearchContext(request, new TimeValue(System.currentTimeMillis()))) { assertEquals( clusterSetting, @@ -1401,10 +1446,22 @@ public void testConcurrentSegmentSearchSearchContext() throws IOException { .get() .getSetting(index, IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey()) ); + + assertEquals( + mode, + client().admin() + .cluster() + .prepareState() + .get() + .getState() + .getMetadata() + .transientSettings() + .get(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), null) + ); searchContext.evaluateRequestShouldUseConcurrentSearch(); assertEquals(concurrentSearchEnabled, searchContext.shouldUseConcurrentSearch()); - // verify executor nullability with concurrent search enabled/disabled - if (concurrentSearchEnabled) { + // verify executor nullability + if (!concurrentSearchExecutorNull) { assertNotNull(searchContext.searcher().getExecutor()); } else { assertNull(searchContext.searcher().getExecutor()); @@ -1415,8 +1472,133 @@ public void testConcurrentSegmentSearchSearchContext() throws IOException { client().admin() .cluster() .prepareUpdateSettings() - .setTransientSettings(Settings.builder().putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey())) + .setTransientSettings( + Settings.builder() + .putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey()) + .putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey()) + ) .get(); + assertSettingDeprecationsAndWarnings(new Setting[] { IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING }); + } + + public void testConcurrentSegmentSearchWithRandomizedModeSettings() throws IOException { + + String index = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); + IndexService indexService = createIndex(index); + final SearchService service = getInstanceFromNode(SearchService.class); + ShardId shardId = new ShardId(indexService.index(), 0); + long nowInMillis = System.currentTimeMillis(); + String clusterAlias = randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.allowPartialSearchResults(randomBoolean()); + ShardSearchRequest request = new ShardSearchRequest( + OriginalIndices.NONE, + searchRequest, + shardId, + indexService.numberOfShards(), + AliasFilter.EMPTY, + 1f, + nowInMillis, + clusterAlias, + Strings.EMPTY_ARRAY + ); + + String[] modeSettings = { "all", "auto", "none", null }; + + // Randomize both index and cluster settings + String clusterMode = randomFrom(modeSettings); + String indexMode = randomFrom(modeSettings); + + // default to false in case mode setting is not set + boolean concurrentSearchEnabled = false; + boolean nullExecutor = true; + + boolean aggregationSupportsConcurrent = randomBoolean(); + + if (indexMode != null) { + concurrentSearchEnabled = !indexMode.equals("none") && aggregationSupportsConcurrent; + nullExecutor = indexMode.equals("none"); + } else if (clusterMode != null) { + concurrentSearchEnabled = !clusterMode.equals("none") && aggregationSupportsConcurrent; + nullExecutor = clusterMode.equals("none"); + } + + // Set the cluster setting for mode + if (clusterMode == null) { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey())) + .get(); + } else { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), clusterMode)) + .get(); + } + + // Set the index setting for mode + if (indexMode == null) { + client().admin() + .indices() + .prepareUpdateSettings(index) + .setSettings(Settings.builder().putNull(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MODE.getKey())) + .get(); + } else { + client().admin() + .indices() + .prepareUpdateSettings(index) + .setSettings(Settings.builder().put(IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MODE.getKey(), indexMode)) + .get(); + } + + try (DefaultSearchContext searchContext = service.createSearchContext(request, new TimeValue(System.currentTimeMillis()))) { + assertEquals( + clusterMode, + client().admin() + .cluster() + .prepareState() + .get() + .getState() + .getMetadata() + .transientSettings() + .get(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey()) + ); + assertEquals( + indexMode, + client().admin() + .indices() + .prepareGetSettings(index) + .get() + .getSetting(index, IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_MODE.getKey()) + ); + SearchContextAggregations mockAggregations = mock(SearchContextAggregations.class); + when(mockAggregations.factories()).thenReturn(mock(AggregatorFactories.class)); + when(mockAggregations.factories().allFactoriesSupportConcurrentSearch()).thenReturn(aggregationSupportsConcurrent); + + // set the aggregations for context + searchContext.aggregations(mockAggregations); + + searchContext.evaluateRequestShouldUseConcurrentSearch(); + // check concurrentSearchenabled based on mode and supportedAggregation is computed correctly + assertEquals(concurrentSearchEnabled, searchContext.shouldUseConcurrentSearch()); + + // Verify executor nullability based on mode + if (!nullExecutor) { + assertNotNull(searchContext.searcher().getExecutor()); + } else { + assertNull(searchContext.searcher().getExecutor()); + } + } + + // Cleanup + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.getKey())) + .get(); + } /** diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 71a2b42c9847a..c439ef209e049 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2285,7 +2285,8 @@ public void onFailure(final Exception e) { responseCollectorService, new NoneCircuitBreakerService(), null, - new TaskResourceTrackingService(settings, clusterSettings, threadPool) + new TaskResourceTrackingService(settings, clusterSettings, threadPool), + Collections.emptyList() ); SearchPhaseController searchPhaseController = new SearchPhaseController( writableRegistry(), diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index ecaee1ccc59b8..09df9b85320f0 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -57,6 +57,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.search.MockSearchService; import org.opensearch.search.SearchService; +import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.query.QueryPhase; import org.opensearch.tasks.TaskResourceTrackingService; @@ -156,7 +157,8 @@ protected SearchService newSearchService( ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, - TaskResourceTrackingService taskResourceTrackingService + TaskResourceTrackingService taskResourceTrackingService, + Collection concurrentSearchDecidersList ) { if (getPluginsService().filterPlugins(MockSearchService.TestPlugin.class).isEmpty()) { return super.newSearchService( @@ -170,7 +172,8 @@ protected SearchService newSearchService( responseCollectorService, circuitBreakerService, indexSearcherExecutor, - taskResourceTrackingService + taskResourceTrackingService, + concurrentSearchDecidersList ); } return new MockSearchService( diff --git a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java index 6c9ace06c8219..28e202e783c4e 100644 --- a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java +++ b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java @@ -45,6 +45,7 @@ import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.threadpool.ThreadPool; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -111,7 +112,8 @@ public MockSearchService( null, circuitBreakerService, indexSearcherExecutor, - taskResourceTrackingService + taskResourceTrackingService, + Collections.emptyList() ); } diff --git a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java index 23316adf6a2d7..ea0331cbfb9a3 100644 --- a/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/ParameterizedOpenSearchIntegTestCase.java @@ -10,7 +10,10 @@ import org.opensearch.common.settings.Settings; +import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; +import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; /** * Base class for running the tests with parameterization of the settings. @@ -35,7 +38,9 @@ abstract class ParameterizedOpenSearchIntegTestCase extends OpenSearchIntegTestC // This method shouldn't be called in setupSuiteScopeCluster(). Only call this method inside single test. public void indexRandomForConcurrentSearch(String... indices) throws InterruptedException { - if (CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.get(settings)) { + if (CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.get(settings) + || CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.get(settings).equals(CONCURRENT_SEGMENT_SEARCH_MODE_AUTO) + || CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE.get(settings).equals(CONCURRENT_SEGMENT_SEARCH_MODE_ALL)) { indexRandomForMultipleSlices(indices); } } From bc02f90540417f63de4b37f3e2bd878d8be00a7f Mon Sep 17 00:00:00 2001 From: Sarat Vemulapalli Date: Tue, 3 Sep 2024 14:39:17 -0700 Subject: [PATCH 032/357] Fixing OpenSearch API Spec annotation build errors (#15626) Signed-off-by: Sarat Vemulapalli --- .../main/java/org/opensearch/wlm/stats/QueryGroupStats.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 9fc7039cd1852..2b389c2167778 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -145,7 +145,7 @@ public static QueryGroupStatsHolder from(QueryGroupState queryGroupState) { } /** - * Writes the {@param statsHolder} to {@param out} + * Writes the @param {statsHolder} to @param {out} * @param out StreamOutput * @param statsHolder QueryGroupStatsHolder * @throws IOException exception @@ -235,7 +235,7 @@ public static ResourceStats from(ResourceTypeState resourceTypeState) { } /** - * Writes the {@param stats} to {@param out} + * Writes the @param {stats} to @param {out} * @param out StreamOutput * @param stats QueryGroupStatsHolder * @throws IOException exception From eb1cbb807298145e40b533d35a70d3e416f7bfc3 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Tue, 3 Sep 2024 15:14:40 -0700 Subject: [PATCH 033/357] Fix flaky test in 91_flat_object_null_value.yml (#15545) This test assumed that the order of returned hits will match the order of insertion. That's not generally true, especially if there was a flush partway through, so documents end up in different segments. This fixes it by explicitly sorting the returned documents to guarantee that they come back in the correct order. Also, we were getting a NPE when trying to output the failure message because the expected value was intentionally null. I fixed that too. Signed-off-by: Michael Froh --- .../test/index/91_flat_object_null_value.yml | 73 +++++++++++++------ .../test/NotEqualMessageBuilder.java | 15 +--- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml index 98abd58a54e4b..716b6fb51cb43 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml @@ -16,12 +16,15 @@ setup: properties: record: type: "flat_object" + order: + type: "integer" - do: index: index: flat_object_null_value id: 1 body: { - "record": null + "record": null, + "order" : 1 } - do: @@ -31,7 +34,8 @@ setup: body: { "record": { "name": null - } + }, + "order" : 2 } - do: @@ -43,7 +47,8 @@ setup: "name": null, "age":"5", "name1": null - } + }, + "order" : 3 } - do: @@ -60,7 +65,8 @@ setup: } } ] - } + }, + "order" : 4 } - do: @@ -77,7 +83,8 @@ setup: }, null ] - } + }, + "order" : 5 } - do: @@ -97,7 +104,8 @@ setup: } } ] - } + }, + "order" : 6 } - do: @@ -108,7 +116,8 @@ setup: "record": { "name": null, "age":"3" - } + }, + "order" : 7 } - do: @@ -119,7 +128,8 @@ setup: "record": { "age":"3", "name": null - } + }, + "order" : 8 } - do: @@ -133,7 +143,8 @@ setup: 3 ], "age": 4 - } + }, + "order" : 9 } - do: @@ -147,7 +158,8 @@ setup: null, 3 ] - } + }, + "order" : 10 } - do: @@ -157,7 +169,8 @@ setup: body: { "record": { "name": null - } + }, + "order": 11 } - do: @@ -171,7 +184,8 @@ setup: null ] } - } + }, + "order": 12 } - do: @@ -183,7 +197,8 @@ setup: "labels": [ null ] - } + }, + "order": 13 } - do: @@ -198,7 +213,8 @@ setup: null ] } - } + }, + "order": 14 } - do: @@ -211,7 +227,8 @@ setup: "labels": [ null ] - } + }, + "order": 15 } - do: @@ -224,7 +241,8 @@ setup: null ], "age": "4" - } + }, + "order": 16 } - do: @@ -239,7 +257,8 @@ setup: "dsdsdsd" ] } - } + }, + "order": 17 } - do: @@ -253,7 +272,8 @@ setup: "name2": null } } - } + }, + "order": 18 } - do: @@ -271,7 +291,8 @@ setup: ] ] } - } + }, + "order": 19 } - do: @@ -299,7 +320,7 @@ teardown: - is_true: flat_object_null_value.mappings - match: { flat_object_null_value.mappings.properties.record.type: flat_object } # https://github.com/opensearch-project/OpenSearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test#length - - length: { flat_object_null_value.mappings.properties: 1 } + - length: { flat_object_null_value.mappings.properties: 2 } --- @@ -328,7 +349,8 @@ teardown: size: 30, query: { exists: { "field": "record" } - } + }, + sort: [{ order: asc}] } - length: { hits.hits: 12 } @@ -352,7 +374,8 @@ teardown: _source: true, query: { exists: { "field": "record.d" } - } + }, + sort: [{ order: asc}] } - length: { hits.hits: 3 } @@ -367,7 +390,8 @@ teardown: _source: true, query: { term: { record: "dsdsdsd" } - } + }, + sort: [{ order: asc}] } - length: { hits.hits: 2 } @@ -381,7 +405,8 @@ teardown: _source: true, query: { term: { record.name.name1: "dsdsdsd" } - } + }, + sort: [{ order: asc}] } - length: { hits.hits: 2 } diff --git a/test/framework/src/main/java/org/opensearch/test/NotEqualMessageBuilder.java b/test/framework/src/main/java/org/opensearch/test/NotEqualMessageBuilder.java index a70e3f15a4bf5..9524b76f255a6 100644 --- a/test/framework/src/main/java/org/opensearch/test/NotEqualMessageBuilder.java +++ b/test/framework/src/main/java/org/opensearch/test/NotEqualMessageBuilder.java @@ -181,18 +181,9 @@ public void compare(String field, boolean hadKey, @Nullable Object actual, Objec field(field, "same [" + expected + "]"); return; } - field( - field, - "expected " - + expected.getClass().getSimpleName() - + " [" - + expected - + "] but was " - + actual.getClass().getSimpleName() - + " [" - + actual - + "]" - ); + String expectedClass = expected == null ? "null object" : expected.getClass().getSimpleName(); + String actualClass = actual == null ? "null object" : actual.getClass().getSimpleName(); + field(field, "expected " + expectedClass + " [" + expected + "] but was " + actualClass + " [" + actual + "]"); } private void indent() { From a59ae3cff98273fb0a10e6dc897b6e1220b8af88 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 4 Sep 2024 05:55:28 +0530 Subject: [PATCH 034/357] Revert "Add fieldType to AbstractQueryBuilder and SortBuilder (#15328)" (#15633) This reverts commit 839ba0bca05f8a9aefdbabd023dc75741265bd6c. Signed-off-by: Ankit Jain --- CHANGELOG.md | 1 - .../index/query/RankFeatureQueryBuilder.java | 5 -- .../join/query/HasChildQueryBuilder.java | 5 -- .../join/query/HasParentQueryBuilder.java | 5 -- .../join/query/ParentIdQueryBuilder.java | 5 -- .../percolator/PercolateQueryBuilder.java | 5 -- .../index/mapper/FieldTypeLookup.java | 3 +- .../index/mapper/MapperService.java | 3 +- .../index/query/AbstractQueryBuilder.java | 23 ------ .../index/query/BoolQueryBuilder.java | 5 -- .../index/query/BoostingQueryBuilder.java | 5 -- .../query/ConstantScoreQueryBuilder.java | 5 -- .../index/query/DisMaxQueryBuilder.java | 5 -- .../query/DistanceFeatureQueryBuilder.java | 3 +- .../index/query/IdsQueryBuilder.java | 5 -- .../index/query/IntervalQueryBuilder.java | 5 -- .../index/query/MatchAllQueryBuilder.java | 5 -- .../index/query/MatchNoneQueryBuilder.java | 5 -- .../index/query/MoreLikeThisQueryBuilder.java | 5 -- .../index/query/MultiMatchQueryBuilder.java | 5 -- .../index/query/NestedQueryBuilder.java | 5 -- .../index/query/QueryShardContext.java | 16 ---- .../index/query/QueryStringQueryBuilder.java | 5 -- .../index/query/ScriptQueryBuilder.java | 5 -- .../index/query/SimpleQueryStringBuilder.java | 5 -- .../query/SpanContainingQueryBuilder.java | 5 -- .../index/query/SpanFirstQueryBuilder.java | 5 -- .../query/SpanMultiTermQueryBuilder.java | 5 -- .../index/query/SpanNearQueryBuilder.java | 5 -- .../index/query/SpanNotQueryBuilder.java | 5 -- .../index/query/SpanOrQueryBuilder.java | 5 -- .../index/query/SpanWithinQueryBuilder.java | 5 -- .../index/query/TermsSetQueryBuilder.java | 5 -- .../index/query/WrapperQueryBuilder.java | 5 -- .../FunctionScoreQueryBuilder.java | 5 -- .../ScriptScoreQueryBuilder.java | 5 -- .../search/sort/FieldSortBuilder.java | 5 -- .../search/sort/ScoreSortBuilder.java | 5 -- .../search/sort/ScriptSortBuilder.java | 5 -- .../opensearch/search/sort/SortBuilder.java | 30 +------- .../index/query/QueryShardContextTests.java | 75 ------------------- .../index/query/plugin/DummyQueryBuilder.java | 5 -- .../opensearch/search/SearchServiceTests.java | 5 -- .../search/sort/AbstractSortTestCase.java | 13 +--- .../search/sort/FieldSortBuilderTests.java | 38 ---------- .../search/sort/plugin/CustomSortBuilder.java | 5 -- 46 files changed, 6 insertions(+), 379 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4e5dd012c2eb..341cc8cf4fc29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) - Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) - Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) -- Add fieldType to AbstractQueryBuilder and FieldSortBuilder ([#15328](https://github.com/opensearch-project/OpenSearch/pull/15328))) - [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410)) - [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java b/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java index 53bafa1346711..13591d0782ea2 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java @@ -400,11 +400,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - @Override protected Query doToQuery(QueryShardContext context) throws IOException { final MappedFieldType ft = context.fieldMapper(field); diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java index 837cf8981fcee..e930780613ed6 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java @@ -264,11 +264,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static HasChildQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String childType = null; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java index e1438f0713034..d296a7b0141ff 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java @@ -233,11 +233,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static HasParentQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String parentType = null; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java index 605f2abcabdc4..bfc01bf151a9c 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java @@ -130,11 +130,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static ParentIdQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String type = null; diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java index 38e941fc42b87..6933bfbef4666 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java @@ -356,11 +356,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, args -> { String field = (String) args[0]; BytesReference document = (BytesReference) args[1]; diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java index dfedbd23ae5ca..7a9a0c5fb5428 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldTypeLookup.java @@ -101,8 +101,7 @@ class FieldTypeLookup implements Iterable { } /** - * Returns the {@link MappedFieldType} for the given field name - * or null if the field name is not found. + * Returns the mapped field type for the given field name. */ public MappedFieldType get(String field) { String concreteField = aliasToConcreteName.getOrDefault(field, field); diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index f75aa0d367ff7..84b0b1d69432d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -632,8 +632,7 @@ public DocumentMapperForType documentMapperWithAutoCreate() { } /** - * Given the full name of a field, returns its {@link MappedFieldType} - * or null if the field is not found. + * Given the full name of a field, returns its {@link MappedFieldType}. */ public MappedFieldType fieldType(String fullName) { return this.mapper == null ? null : this.mapper.fieldTypes().get(fullName); diff --git a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java index 2e9b856b4fdc5..66c6ee115c3f0 100644 --- a/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java @@ -74,7 +74,6 @@ public abstract class AbstractQueryBuilder> public static final ParseField BOOST_FIELD = new ParseField("boost"); protected String queryName; - protected String fieldType; protected float boost = DEFAULT_BOOST; protected AbstractQueryBuilder() { @@ -113,27 +112,6 @@ protected void printBoostAndQueryName(XContentBuilder builder) throws IOExceptio } } - /** - * Returns field name as String. - * Abstract method to be implemented by all child classes. - */ - public abstract String fieldName(); - - /** - * Default method for child classes which do not have a custom {@link #fieldName()} implementation. - */ - protected static String getDefaultFieldName() { - return null; - }; - - /** - * Returns field type as String for QueryBuilder classes which have a defined fieldName. - * Else returns null. - */ - public final String getFieldType() { - return fieldType; - }; - @Override public final Query toQuery(QueryShardContext context) throws IOException { Query query = doToQuery(context); @@ -147,7 +125,6 @@ public final Query toQuery(QueryShardContext context) throws IOException { context.addNamedQuery(queryName, query); } } - fieldType = context.getFieldTypeString(fieldName()); return query; } diff --git a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java index fe93a5aec6950..c44a7ef6a397c 100644 --- a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java @@ -270,11 +270,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - private static void doXArrayContent(ParseField field, List clauses, XContentBuilder builder, Params params) throws IOException { if (clauses.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java index 185dbf60b5a43..1b52ae2f03605 100644 --- a/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoostingQueryBuilder.java @@ -151,11 +151,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static BoostingQueryBuilder fromXContent(XContentParser parser) throws IOException { QueryBuilder positiveQuery = null; boolean positiveQueryFound = false; diff --git a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java index db92536fe9201..b2764d29da80a 100644 --- a/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java @@ -101,11 +101,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static ConstantScoreQueryBuilder fromXContent(XContentParser parser) throws IOException { QueryBuilder query = null; boolean queryFound = false; diff --git a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java index fe90be91a8f89..bd8ec62f6c43e 100644 --- a/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DisMaxQueryBuilder.java @@ -137,11 +137,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static DisMaxQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; float tieBreaker = DisMaxQueryBuilder.DEFAULT_TIE_BREAKER; diff --git a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java index 016b6357f4934..1d9f0479c6b17 100644 --- a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java @@ -136,8 +136,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { return fieldType.distanceFeatureQuery(origin.origin(), pivot, 1.0f, context); } - @Override - public String fieldName() { + String fieldName() { return field; } diff --git a/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java index a47b37204fa55..d7ebdbff10adb 100644 --- a/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/IdsQueryBuilder.java @@ -132,11 +132,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - private static final ObjectParser PARSER = new ObjectParser<>(NAME, IdsQueryBuilder::new); static { diff --git a/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java index 3965bb47ad1b9..125035ea5e95a 100644 --- a/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/IntervalQueryBuilder.java @@ -95,11 +95,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static IntervalQueryBuilder fromXContent(XContentParser parser) throws IOException { if (parser.nextToken() != XContentParser.Token.FIELD_NAME) { throw new ParsingException(parser.getTokenLocation(), "Expected [FIELD_NAME] but got [" + parser.currentToken() + "]"); diff --git a/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java index 07e6737597977..c62ee0ac39584 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchAllQueryBuilder.java @@ -72,11 +72,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - private static final ObjectParser PARSER = new ObjectParser<>(NAME, MatchAllQueryBuilder::new); static { diff --git a/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java index c07e5c9d77600..17e84bc785206 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchNoneQueryBuilder.java @@ -71,11 +71,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static MatchNoneQueryBuilder fromXContent(XContentParser parser) throws IOException { String currentFieldName = null; XContentParser.Token token; diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java index 141d4beb5f885..e6472afef2215 100644 --- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java @@ -792,11 +792,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static MoreLikeThisQueryBuilder fromXContent(XContentParser parser) throws IOException { // document inputs List fields = null; diff --git a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java index 8eb5beadf3e48..6227e5d2fa806 100644 --- a/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MultiMatchQueryBuilder.java @@ -631,11 +631,6 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws IOException { Object value = null; Map fieldsBoosts = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java index f5aab8c967b88..5908882472ce7 100644 --- a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java @@ -205,11 +205,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static NestedQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; ScoreMode scoreMode = ScoreMode.Avg; diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index b0ed049e6a1f4..91313092d8d28 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -364,22 +364,6 @@ public MappedFieldType fieldMapper(String name) { return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); } - /** - * Returns field type as String for the given field name. - * If field is not mapped or mapperService is null, returns null. - */ - public String getFieldTypeString(String fieldName) { - if (fieldName != null) { - if (mapperService != null) { - MappedFieldType mappedFieldType = mapperService.fieldType(fieldName); - if (mappedFieldType != null) { - return mappedFieldType.typeName(); - } - } - } - return null; - } - public ObjectMapper getObjectMapper(String name) { return mapperService.getObjectMapper(name); } diff --git a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java index e6148a23944d9..3d8fbd5fc436d 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/QueryStringQueryBuilder.java @@ -647,11 +647,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static QueryStringQueryBuilder fromXContent(XContentParser parser) throws IOException { String currentFieldName = null; XContentParser.Token token; diff --git a/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java index 4d895bc8bb725..ded6fd0528c33 100644 --- a/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ScriptQueryBuilder.java @@ -106,11 +106,6 @@ protected void doXContent(XContentBuilder builder, Params builderParams) throws builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static ScriptQueryBuilder fromXContent(XContentParser parser) throws IOException { // also, when caching, since its isCacheable is false, will result in loading all bit set... Script script = null; diff --git a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java index 5d5da6275601c..57ae7dd0ea5e9 100644 --- a/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SimpleQueryStringBuilder.java @@ -479,11 +479,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SimpleQueryStringBuilder fromXContent(XContentParser parser) throws IOException { String currentFieldName = null; String queryBody = null; diff --git a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java index e7cc3c6f40855..32a19ea3e9b50 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanContainingQueryBuilder.java @@ -117,11 +117,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanContainingQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; diff --git a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java index d8ad0c83ad0bd..bcbc64ddf386d 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanFirstQueryBuilder.java @@ -120,11 +120,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanFirstQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; diff --git a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java index f026dddfe3522..96d03c91964e3 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanMultiTermQueryBuilder.java @@ -99,11 +99,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanMultiTermQueryBuilder fromXContent(XContentParser parser) throws IOException { String currentFieldName = null; MultiTermQueryBuilder subQuery = null; diff --git a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java index 536024666dc38..30a1c29c29126 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java @@ -167,11 +167,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanNearQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; int slop = DEFAULT_SLOP; diff --git a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java index 75ccd028656c2..59ec5b9d77fc8 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNotQueryBuilder.java @@ -181,11 +181,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanNotQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; diff --git a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java index 2014d9b807b6f..fae1e318c66bd 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanOrQueryBuilder.java @@ -115,11 +115,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanOrQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; diff --git a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java index b1c0aebc1fb52..4d5a6dde61a70 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanWithinQueryBuilder.java @@ -122,11 +122,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static SpanWithinQueryBuilder fromXContent(XContentParser parser) throws IOException { float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; diff --git a/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java index d6b3ecd102c08..e2cf7384ecac7 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsSetQueryBuilder.java @@ -169,11 +169,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static TermsSetQueryBuilder fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { diff --git a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java index d08d50f17191b..8a322b2f9e173 100644 --- a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java @@ -131,11 +131,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static WrapperQueryBuilder fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java index 454bf62e837f9..b3c797f11de6d 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/FunctionScoreQueryBuilder.java @@ -300,11 +300,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public FunctionScoreQueryBuilder setMinScore(float minScore) { this.minScore = minScore; return this; diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java index 160616faed114..fe9ad200d44f0 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/ScriptScoreQueryBuilder.java @@ -155,11 +155,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public ScriptScoreQueryBuilder setMinScore(float minScore) { this.minScore = minScore; return this; diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index c93e37da6854d..5cecda1346b90 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -559,11 +559,6 @@ public BucketedSort buildBucketedSort(QueryShardContext context, int bucketSize, } } - @Override - public String fieldName() { - return fieldName; - } - private MappedFieldType resolveUnmappedType(QueryShardContext context) { if (unmappedType == null) { throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on"); diff --git a/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java index 38774c0e10b71..1be49e5ca81ce 100644 --- a/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/ScoreSortBuilder.java @@ -160,11 +160,6 @@ protected float docValue() { }; } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - @Override public boolean equals(Object object) { if (this == object) { diff --git a/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java index 0ffc2b960fcd3..bb1930eb3a953 100644 --- a/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/ScriptSortBuilder.java @@ -335,11 +335,6 @@ public BucketedSort buildBucketedSort(QueryShardContext context, int bucketSize, return fieldComparatorSource(context).newBucketedSort(context.bigArrays(), order, DocValueFormat.RAW, bucketSize, extra); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - private IndexFieldData.XFieldComparatorSource fieldComparatorSource(QueryShardContext context) throws IOException { MultiValueMode valueMode = null; if (sortMode != null) { diff --git a/server/src/main/java/org/opensearch/search/sort/SortBuilder.java b/server/src/main/java/org/opensearch/search/sort/SortBuilder.java index d6c5f2b5f3144..a8c21e7311061 100644 --- a/server/src/main/java/org/opensearch/search/sort/SortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/SortBuilder.java @@ -71,7 +71,6 @@ public abstract class SortBuilder> implements NamedWriteable, ToXContentObject, Rewriteable> { protected SortOrder order = SortOrder.ASC; - protected String fieldType; // parse fields common to more than one SortBuilder public static final ParseField ORDER_FIELD = new ParseField("order"); @@ -162,17 +161,11 @@ private static void parseCompoundSortField(XContentParser parser, List buildSort(List> sortBuilders, QueryShardContext context) throws IOException { List sortFields = new ArrayList<>(sortBuilders.size()); List sortFormats = new ArrayList<>(sortBuilders.size()); for (SortBuilder builder : sortBuilders) { - SortFieldAndFormat sf = builder.buildHelper(context); + SortFieldAndFormat sf = builder.build(context); sortFields.add(sf.field); sortFormats.add(sf.format); } @@ -294,25 +287,4 @@ protected static QueryBuilder parseNestedFilter(XContentParser parser) { public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this, true, true); } - - /** - * Returns field name as String. - * Abstract method to be implemented by all child classes. - */ - public abstract String fieldName(); - - /** - * Default method for child classes which do not have a custom {@link #fieldName()} implementation. - */ - protected static String getDefaultFieldName() { - return null; - }; - - /** - * Returns field type as String for SortBuilder classes which have a defined fieldName. - * Else returns null. - */ - public final String getFieldType() { - return fieldType; - }; } diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java index 294e175ca670e..12677edc8efa7 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java @@ -69,8 +69,6 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.TextFieldMapper; -import org.opensearch.index.mapper.TextSearchInfo; -import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.search.lookup.LeafDocLookup; import org.opensearch.search.lookup.LeafSearchLookup; import org.opensearch.search.lookup.SearchLookup; @@ -152,11 +150,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } - @Override - public String fieldName() { - return getDefaultFieldName(); - } - @Override protected Query doToQuery(QueryShardContext context) throws IOException { throw new RuntimeException("boom"); @@ -486,72 +479,4 @@ public void collect(int doc) throws IOException { } } - public void testGetFieldTypeString() { - MapperService mapperService = mock(MapperService.class); - - Settings settings = Settings.builder() - .put("index.version.created", Version.CURRENT) - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build(); - IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - QueryShardContext queryShardContext = new QueryShardContext( - 0, - indexSettings, - null, - null, - null, - mapperService, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ); - - MappedFieldType mappedFieldType = new MappedFieldType("field", true, false, true, TextSearchInfo.NONE, Collections.emptyMap()) { - @Override - public String typeName() { - return "long"; - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - return null; - } - }; - - when(mapperService.fieldType(any())).thenReturn(null); - - // Null fieldName - String fieldType = queryShardContext.getFieldTypeString(null); - assertEquals(null, fieldType); - - // MapperService return null - fieldType = queryShardContext.getFieldTypeString("field1"); - assertEquals(null, fieldType); - - when(mapperService.fieldType("field")).thenReturn(mappedFieldType); - - // Unmapped fieldName - fieldType = queryShardContext.getFieldTypeString("unknown_field"); - assertEquals(null, fieldType); - // Date fieldType - fieldType = queryShardContext.getFieldTypeString("field"); - assertEquals("long", fieldType); - } - } diff --git a/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java b/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java index a32b8aaf7c2f7..2b0aaa2dd7091 100644 --- a/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java +++ b/server/src/test/java/org/opensearch/index/query/plugin/DummyQueryBuilder.java @@ -62,11 +62,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.startObject(NAME).endObject(); } - @Override - public final String fieldName() { - return getDefaultFieldName(); - } - public static DummyQueryBuilder fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); assert token == XContentParser.Token.END_OBJECT; diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java index 41c77866f8167..514e99a126267 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java @@ -905,11 +905,6 @@ protected void doWriteTo(StreamOutput out) {} @Override protected void doXContent(XContentBuilder builder, Params params) {} - @Override - public String fieldName() { - return getDefaultFieldName(); - } - @Override protected Query doToQuery(QueryShardContext context) { return null; diff --git a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java index 9634c80f26fb8..257ff1015e3b4 100644 --- a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java @@ -55,7 +55,6 @@ import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.Mapper.BuilderContext; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.index.mapper.ObjectMapper.Nested; @@ -200,18 +199,10 @@ public void testEqualsAndHashcode() { } protected final QueryShardContext createMockShardContext() { - return createMockShardContext(null, null); + return createMockShardContext(null); } protected final QueryShardContext createMockShardContext(IndexSearcher searcher) { - return createMockShardContext(searcher, null); - } - - protected final QueryShardContext createMockShardContext(MapperService mockMapperService) { - return createMockShardContext(null, mockMapperService); - } - - protected final QueryShardContext createMockShardContext(IndexSearcher searcher, MapperService mapperService) { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings( index, @@ -231,7 +222,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - mapperService, + null, null, scriptService, xContentRegistry(), diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java index 3a889fa0bc551..9b8cd1b5f1ce0 100644 --- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java @@ -46,7 +46,6 @@ import org.apache.lucene.sandbox.document.BigIntegerPoint; import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSelector; import org.apache.lucene.search.SortedNumericSortField; @@ -65,11 +64,8 @@ import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NestedPathFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.mapper.TextSearchInfo; -import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -81,13 +77,11 @@ import org.opensearch.search.MultiValueMode; import org.opensearch.search.SearchSortValuesAndFormats; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -97,8 +91,6 @@ import static org.opensearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.opensearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -712,36 +704,6 @@ public void testIsBottomSortShardDisjoint() throws Exception { } } - /** - * Test that the sort builder fieldType is set properly - */ - public void testSortFieldFieldType() throws IOException { - MapperService mapperService = mock(MapperService.class); - QueryShardContext shardContextMock = createMockShardContext(mapperService); - - MappedFieldType mappedFieldType = new MappedFieldType("field", true, false, true, TextSearchInfo.NONE, Collections.emptyMap()) { - @Override - public String typeName() { - return "double"; - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - return null; - } - }; - - when(mapperService.fieldType("field")).thenReturn(mappedFieldType); - FieldSortBuilder fieldSortBuilder = new FieldSortBuilder("field"); - fieldSortBuilder.buildHelper(shardContextMock); - assertEquals(fieldSortBuilder.getFieldType(), "double"); - } - @Override protected void assertWarnings(FieldSortBuilder testItem) { List expectedWarnings = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java b/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java index 56b7b9d86c253..c3d790cb99cbd 100644 --- a/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java +++ b/server/src/test/java/org/opensearch/search/sort/plugin/CustomSortBuilder.java @@ -75,11 +75,6 @@ public BucketedSort buildBucketedSort(final QueryShardContext context, final int throw new IllegalStateException("rewrite"); } - @Override - public String fieldName() { - return field; - } - @Override public boolean equals(Object object) { if (this == object) { From 0753461986cd5746bf7b7ed2ba153cc65cea9a59 Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Tue, 3 Sep 2024 18:36:19 -0700 Subject: [PATCH 035/357] Add Update QueryGroup API Logic (#14775) * Add Update QueryGroup API Logic Signed-off-by: Ruirui Zhang * append to changlog Signed-off-by: Ruirui Zhang * add javadoc Signed-off-by: Ruirui Zhang * rebase Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * fix UT Signed-off-by: Ruirui Zhang * adress comments Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + .../plugin/wlm/WorkloadManagementPlugin.java | 13 +- .../wlm/action/CreateQueryGroupRequest.java | 4 +- .../TransportUpdateQueryGroupAction.java | 51 ++++ .../wlm/action/UpdateQueryGroupAction.java | 36 +++ .../wlm/action/UpdateQueryGroupRequest.java | 83 +++++++ .../wlm/action/UpdateQueryGroupResponse.java | 74 ++++++ .../wlm/rest/RestGetQueryGroupAction.java | 2 +- .../wlm/rest/RestUpdateQueryGroupAction.java | 72 ++++++ .../service/QueryGroupPersistenceService.java | 139 ++++++++--- .../plugin/wlm/QueryGroupTestUtils.java | 40 +++- .../action/CreateQueryGroupRequestTests.java | 2 +- .../action/CreateQueryGroupResponseTests.java | 6 +- .../action/GetQueryGroupResponseTests.java | 16 +- .../wlm/action/QueryGroupActionTestUtils.java | 17 ++ .../action/UpdateQueryGroupRequestTests.java | 97 ++++++++ .../action/UpdateQueryGroupResponseTests.java | 67 ++++++ .../QueryGroupPersistenceServiceTests.java | 182 +++++++++++++- .../api/update_query_group_context.json | 23 ++ .../rest-api-spec/test/wlm/10_query_group.yml | 42 ++++ .../cluster/metadata/QueryGroup.java | 165 +++++-------- .../wlm/MutableQueryGroupFragment.java | 225 ++++++++++++++++++ .../metadata/QueryGroupMetadataTests.java | 6 +- .../cluster/metadata/QueryGroupTests.java | 64 +++-- .../wlm/MutableQueryGroupFragmentTests.java | 66 +++++ 25 files changed, 1292 insertions(+), 201 deletions(-) create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupAction.java create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponse.java create mode 100644 plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateQueryGroupAction.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/QueryGroupActionTestUtils.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequestTests.java create mode 100644 plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponseTests.java create mode 100644 plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/api/update_query_group_context.json create mode 100644 server/src/main/java/org/opensearch/wlm/MutableQueryGroupFragment.java create mode 100644 server/src/test/java/org/opensearch/wlm/MutableQueryGroupFragmentTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 341cc8cf4fc29..59bf2c1ae2b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) - [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709)) - [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028)) +- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775)) - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) - Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java index 64f510fa1db67..c86490552f2f2 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java @@ -24,9 +24,12 @@ import org.opensearch.plugin.wlm.action.TransportCreateQueryGroupAction; import org.opensearch.plugin.wlm.action.TransportDeleteQueryGroupAction; import org.opensearch.plugin.wlm.action.TransportGetQueryGroupAction; +import org.opensearch.plugin.wlm.action.TransportUpdateQueryGroupAction; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupAction; import org.opensearch.plugin.wlm.rest.RestCreateQueryGroupAction; import org.opensearch.plugin.wlm.rest.RestDeleteQueryGroupAction; import org.opensearch.plugin.wlm.rest.RestGetQueryGroupAction; +import org.opensearch.plugin.wlm.rest.RestUpdateQueryGroupAction; import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.Plugin; @@ -52,7 +55,8 @@ public WorkloadManagementPlugin() {} return List.of( new ActionPlugin.ActionHandler<>(CreateQueryGroupAction.INSTANCE, TransportCreateQueryGroupAction.class), new ActionPlugin.ActionHandler<>(GetQueryGroupAction.INSTANCE, TransportGetQueryGroupAction.class), - new ActionPlugin.ActionHandler<>(DeleteQueryGroupAction.INSTANCE, TransportDeleteQueryGroupAction.class) + new ActionPlugin.ActionHandler<>(DeleteQueryGroupAction.INSTANCE, TransportDeleteQueryGroupAction.class), + new ActionPlugin.ActionHandler<>(UpdateQueryGroupAction.INSTANCE, TransportUpdateQueryGroupAction.class) ); } @@ -66,7 +70,12 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestCreateQueryGroupAction(), new RestGetQueryGroupAction(), new RestDeleteQueryGroupAction()); + return List.of( + new RestCreateQueryGroupAction(), + new RestGetQueryGroupAction(), + new RestDeleteQueryGroupAction(), + new RestUpdateQueryGroupAction() + ); } @Override diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java index ff6422be36885..d92283391dd3b 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java @@ -40,7 +40,7 @@ public class CreateQueryGroupRequest extends ActionRequest { * Constructor for CreateQueryGroupRequest * @param queryGroup - A {@link QueryGroup} object */ - public CreateQueryGroupRequest(QueryGroup queryGroup) { + CreateQueryGroupRequest(QueryGroup queryGroup) { this.queryGroup = queryGroup; } @@ -48,7 +48,7 @@ public CreateQueryGroupRequest(QueryGroup queryGroup) { * Constructor for CreateQueryGroupRequest * @param in - A {@link StreamInput} object */ - public CreateQueryGroupRequest(StreamInput in) throws IOException { + CreateQueryGroupRequest(StreamInput in) throws IOException { super(in); queryGroup = new QueryGroup(in); } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java new file mode 100644 index 0000000000000..a6aa2da8fdc08 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +/** + * Transport action to update QueryGroup + * + * @opensearch.experimental + */ +public class TransportUpdateQueryGroupAction extends HandledTransportAction { + + private final QueryGroupPersistenceService queryGroupPersistenceService; + + /** + * Constructor for TransportUpdateQueryGroupAction + * + * @param actionName - action name + * @param transportService - a {@link TransportService} object + * @param actionFilters - a {@link ActionFilters} object + * @param queryGroupPersistenceService - a {@link QueryGroupPersistenceService} object + */ + @Inject + public TransportUpdateQueryGroupAction( + String actionName, + TransportService transportService, + ActionFilters actionFilters, + QueryGroupPersistenceService queryGroupPersistenceService + ) { + super(UpdateQueryGroupAction.NAME, transportService, actionFilters, UpdateQueryGroupRequest::new); + this.queryGroupPersistenceService = queryGroupPersistenceService; + } + + @Override + protected void doExecute(Task task, UpdateQueryGroupRequest request, ActionListener listener) { + queryGroupPersistenceService.updateInClusterStateMetadata(request, listener); + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupAction.java new file mode 100644 index 0000000000000..ff472f206131c --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupAction.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.action.ActionType; + +/** + * Transport action to update QueryGroup + * + * @opensearch.experimental + */ +public class UpdateQueryGroupAction extends ActionType { + + /** + * An instance of UpdateQueryGroupAction + */ + public static final UpdateQueryGroupAction INSTANCE = new UpdateQueryGroupAction(); + + /** + * Name for UpdateQueryGroupAction + */ + public static final String NAME = "cluster:admin/opensearch/wlm/query_group/_update"; + + /** + * Default constructor + */ + private UpdateQueryGroupAction() { + super(NAME, UpdateQueryGroupResponse::new); + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java new file mode 100644 index 0000000000000..048b599f095fd --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.wlm.MutableQueryGroupFragment; + +import java.io.IOException; + +/** + * A request for update QueryGroup + * + * @opensearch.experimental + */ +public class UpdateQueryGroupRequest extends ActionRequest { + private final String name; + private final MutableQueryGroupFragment mutableQueryGroupFragment; + + /** + * Constructor for UpdateQueryGroupRequest + * @param name - QueryGroup name for UpdateQueryGroupRequest + * @param mutableQueryGroupFragment - MutableQueryGroupFragment for UpdateQueryGroupRequest + */ + UpdateQueryGroupRequest(String name, MutableQueryGroupFragment mutableQueryGroupFragment) { + this.name = name; + this.mutableQueryGroupFragment = mutableQueryGroupFragment; + } + + /** + * Constructor for UpdateQueryGroupRequest + * @param in - A {@link StreamInput} object + */ + UpdateQueryGroupRequest(StreamInput in) throws IOException { + this(in.readString(), new MutableQueryGroupFragment(in)); + } + + /** + * Generate a UpdateQueryGroupRequest from XContent + * @param parser - A {@link XContentParser} object + * @param name - name of the QueryGroup to be updated + */ + public static UpdateQueryGroupRequest fromXContent(XContentParser parser, String name) throws IOException { + QueryGroup.Builder builder = QueryGroup.Builder.fromXContent(parser); + return new UpdateQueryGroupRequest(name, builder.getMutableQueryGroupFragment()); + } + + @Override + public ActionRequestValidationException validate() { + QueryGroup.validateName(name); + return null; + } + + /** + * name getter + */ + public String getName() { + return name; + } + + /** + * mutableQueryGroupFragment getter + */ + public MutableQueryGroupFragment getmMutableQueryGroupFragment() { + return mutableQueryGroupFragment; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + mutableQueryGroupFragment.writeTo(out); + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponse.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponse.java new file mode 100644 index 0000000000000..9071f52ecb5a7 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponse.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Response for the update API for QueryGroup + * + * @opensearch.experimental + */ +public class UpdateQueryGroupResponse extends ActionResponse implements ToXContent, ToXContentObject { + private final QueryGroup queryGroup; + private final RestStatus restStatus; + + /** + * Constructor for UpdateQueryGroupResponse + * @param queryGroup - the QueryGroup to be updated + * @param restStatus - the rest status for the response + */ + public UpdateQueryGroupResponse(final QueryGroup queryGroup, RestStatus restStatus) { + this.queryGroup = queryGroup; + this.restStatus = restStatus; + } + + /** + * Constructor for UpdateQueryGroupResponse + * @param in - a {@link StreamInput} object + */ + public UpdateQueryGroupResponse(StreamInput in) throws IOException { + queryGroup = new QueryGroup(in); + restStatus = RestStatus.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + queryGroup.writeTo(out); + RestStatus.writeTo(out, restStatus); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return queryGroup.toXContent(builder, params); + } + + /** + * queryGroup getter + */ + public QueryGroup getQueryGroup() { + return queryGroup; + } + + /** + * restStatus getter + */ + public RestStatus getRestStatus() { + return restStatus; + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestGetQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestGetQueryGroupAction.java index c250bd2979e98..c87973e113138 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestGetQueryGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestGetQueryGroupAction.java @@ -27,7 +27,7 @@ import static org.opensearch.rest.RestRequest.Method.GET; /** - * Rest action to get a QueryGroup0 + * Rest action to get a QueryGroup * * @opensearch.experimental */ diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateQueryGroupAction.java new file mode 100644 index 0000000000000..55b4bc5a295c4 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rest/RestUpdateQueryGroupAction.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.rest; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupAction; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupRequest; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupResponse; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; + +import java.io.IOException; +import java.util.List; + +import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; + +/** + * Rest action to update a QueryGroup + * + * @opensearch.experimental + */ +public class RestUpdateQueryGroupAction extends BaseRestHandler { + + /** + * Constructor for RestUpdateQueryGroupAction + */ + public RestUpdateQueryGroupAction() {} + + @Override + public String getName() { + return "update_query_group"; + } + + /** + * The list of {@link Route}s that this RestHandler is responsible for handling. + */ + @Override + public List routes() { + return List.of(new Route(POST, "_wlm/query_group/{name}"), new Route(PUT, "_wlm/query_group/{name}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + try (XContentParser parser = request.contentParser()) { + UpdateQueryGroupRequest updateQueryGroupRequest = UpdateQueryGroupRequest.fromXContent(parser, request.param("name")); + return channel -> client.execute(UpdateQueryGroupAction.INSTANCE, updateQueryGroupRequest, updateQueryGroupResponse(channel)); + } + } + + private RestResponseListener updateQueryGroupResponse(final RestChannel channel) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(final UpdateQueryGroupResponse response) throws Exception { + return new BytesRestResponse(RestStatus.OK, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + } + }; + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java index 7561a2f6f99c3..f9332ff3022dc 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java @@ -29,6 +29,9 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse; import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupRequest; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupResponse; +import org.opensearch.wlm.MutableQueryGroupFragment; import org.opensearch.wlm.ResourceType; import java.util.Collection; @@ -37,6 +40,8 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.opensearch.cluster.metadata.QueryGroup.updateExistingQueryGroup; + /** * This class defines the functions for QueryGroup persistence */ @@ -44,6 +49,7 @@ public class QueryGroupPersistenceService { static final String SOURCE = "query-group-persistence-service"; private static final String CREATE_QUERY_GROUP_THROTTLING_KEY = "create-query-group"; private static final String DELETE_QUERY_GROUP_THROTTLING_KEY = "delete-query-group"; + private static final String UPDATE_QUERY_GROUP_THROTTLING_KEY = "update-query-group"; private static final Logger logger = LogManager.getLogger(QueryGroupPersistenceService.class); /** * max QueryGroup count setting name @@ -72,6 +78,7 @@ public class QueryGroupPersistenceService { private volatile int maxQueryGroupCount; final ThrottlingKey createQueryGroupThrottlingKey; final ThrottlingKey deleteQueryGroupThrottlingKey; + final ThrottlingKey updateQueryGroupThrottlingKey; /** * Constructor for QueryGroupPersistenceService @@ -89,6 +96,7 @@ public QueryGroupPersistenceService( this.clusterService = clusterService; this.createQueryGroupThrottlingKey = clusterService.registerClusterManagerTask(CREATE_QUERY_GROUP_THROTTLING_KEY, true); this.deleteQueryGroupThrottlingKey = clusterService.registerClusterManagerTask(DELETE_QUERY_GROUP_THROTTLING_KEY, true); + this.updateQueryGroupThrottlingKey = clusterService.registerClusterManagerTask(UPDATE_QUERY_GROUP_THROTTLING_KEY, true); setMaxQueryGroupCount(MAX_QUERY_GROUP_COUNT.get(settings)); clusterSettings.addSettingsUpdateConsumer(MAX_QUERY_GROUP_COUNT, this::setMaxQueryGroupCount); } @@ -169,39 +177,13 @@ ClusterState saveQueryGroupInClusterState(final QueryGroup queryGroup, final Clu } // check if there's any resource allocation that exceed limit of 1.0 - Map totalUsageMap = calculateTotalUsage(existingQueryGroups, queryGroup); - for (ResourceType resourceType : queryGroup.getResourceLimits().keySet()) { - if (totalUsageMap.get(resourceType) > 1) { - logger.warn("Total resource allocation for {} will go above the max limit of 1.0.", resourceType.getName()); - throw new IllegalArgumentException( - "Total resource allocation for " + resourceType.getName() + " will go above the max limit of 1.0." - ); - } - } + validateTotalUsage(existingQueryGroups, groupName, queryGroup.getResourceLimits()); return ClusterState.builder(currentClusterState) .metadata(Metadata.builder(currentClusterState.metadata()).put(queryGroup).build()) .build(); } - /** - * This method calculates the existing total usage of the all the resource limits - * @param existingQueryGroups - existing QueryGroups in the system - * @param queryGroup - the QueryGroup we're creating or updating - */ - private Map calculateTotalUsage(Map existingQueryGroups, QueryGroup queryGroup) { - final Map map = new EnumMap<>(ResourceType.class); - map.putAll(queryGroup.getResourceLimits()); - for (QueryGroup currGroup : existingQueryGroups.values()) { - if (!currGroup.getName().equals(queryGroup.getName())) { - for (ResourceType resourceType : queryGroup.getResourceLimits().keySet()) { - map.compute(resourceType, (k, v) -> v + currGroup.getResourceLimits().get(resourceType)); - } - } - } - return map; - } - /** * Get the QueryGroups with the specified name from cluster state * @param name - the QueryGroup name we are getting @@ -264,10 +246,113 @@ ClusterState deleteQueryGroupInClusterState(final String name, final ClusterStat return ClusterState.builder(currentClusterState).metadata(Metadata.builder(metadata).remove(queryGroupToRemove).build()).build(); } + /** + * Modify cluster state to update the QueryGroup + * @param toUpdateGroup {@link QueryGroup} - the QueryGroup that we want to update + * @param listener - ActionListener for UpdateQueryGroupResponse + */ + public void updateInClusterStateMetadata(UpdateQueryGroupRequest toUpdateGroup, ActionListener listener) { + clusterService.submitStateUpdateTask(SOURCE, new ClusterStateUpdateTask(Priority.NORMAL) { + @Override + public ClusterState execute(ClusterState currentState) { + return updateQueryGroupInClusterState(toUpdateGroup, currentState); + } + + @Override + public ThrottlingKey getClusterManagerThrottlingKey() { + return updateQueryGroupThrottlingKey; + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn("Failed to update QueryGroup due to error: {}, for source: {}", e.getMessage(), source); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + String name = toUpdateGroup.getName(); + Optional findUpdatedGroup = newState.metadata() + .queryGroups() + .values() + .stream() + .filter(group -> group.getName().equals(name)) + .findFirst(); + assert findUpdatedGroup.isPresent(); + QueryGroup updatedGroup = findUpdatedGroup.get(); + UpdateQueryGroupResponse response = new UpdateQueryGroupResponse(updatedGroup, RestStatus.OK); + listener.onResponse(response); + } + }); + } + + /** + * Modify cluster state to update the existing QueryGroup + * @param updateQueryGroupRequest {@link QueryGroup} - the QueryGroup that we want to update + * @param currentState - current cluster state + */ + ClusterState updateQueryGroupInClusterState(UpdateQueryGroupRequest updateQueryGroupRequest, ClusterState currentState) { + final Metadata metadata = currentState.metadata(); + final Map existingGroups = currentState.metadata().queryGroups(); + String name = updateQueryGroupRequest.getName(); + MutableQueryGroupFragment mutableQueryGroupFragment = updateQueryGroupRequest.getmMutableQueryGroupFragment(); + + final QueryGroup existingGroup = existingGroups.values() + .stream() + .filter(group -> group.getName().equals(name)) + .findFirst() + .orElseThrow(() -> new ResourceNotFoundException("No QueryGroup exists with the provided name: " + name)); + + validateTotalUsage(existingGroups, name, mutableQueryGroupFragment.getResourceLimits()); + return ClusterState.builder(currentState) + .metadata( + Metadata.builder(metadata) + .remove(existingGroup) + .put(updateExistingQueryGroup(existingGroup, mutableQueryGroupFragment)) + .build() + ) + .build(); + } + + /** + * This method checks if there's any resource allocation that exceed limit of 1.0 + * @param existingQueryGroups - existing QueryGroups in the system + * @param resourceLimits - the QueryGroup we're creating or updating + */ + private void validateTotalUsage(Map existingQueryGroups, String name, Map resourceLimits) { + if (resourceLimits == null || resourceLimits.isEmpty()) { + return; + } + final Map totalUsage = new EnumMap<>(ResourceType.class); + totalUsage.putAll(resourceLimits); + for (QueryGroup currGroup : existingQueryGroups.values()) { + if (!currGroup.getName().equals(name)) { + for (ResourceType resourceType : resourceLimits.keySet()) { + totalUsage.compute(resourceType, (k, v) -> v + currGroup.getResourceLimits().getOrDefault(resourceType, 0.0)); + } + } + } + totalUsage.forEach((resourceType, total) -> { + if (total > 1.0) { + logger.warn("Total resource allocation for {} will go above the max limit of 1.0.", resourceType.getName()); + throw new IllegalArgumentException( + "Total resource allocation for " + resourceType.getName() + " will go above the max limit of 1.0." + ); + } + }); + } + /** * maxQueryGroupCount getter */ public int getMaxQueryGroupCount() { return maxQueryGroupCount; } + + /** + * clusterService getter + */ + public ClusterService getClusterService() { + return clusterService; + } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java index e165645775d5c..c6eb3140e943d 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java @@ -21,6 +21,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.ResourceType; import java.util.ArrayList; import java.util.Collection; @@ -31,7 +33,6 @@ import java.util.Set; import static org.opensearch.cluster.metadata.QueryGroup.builder; -import static org.opensearch.wlm.ResourceType.fromName; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -43,21 +44,21 @@ public class QueryGroupTestUtils { public static final String _ID_ONE = "AgfUO5Ja9yfsYlONlYi3TQ=="; public static final String _ID_TWO = "G5iIqHy4g7eK1qIAAAAIH53=1"; public static final String NAME_NONE_EXISTED = "query_group_none_existed"; - public static final String MEMORY_STRING = "memory"; - public static final String MONITOR_STRING = "monitor"; public static final long TIMESTAMP_ONE = 4513232413L; public static final long TIMESTAMP_TWO = 4513232415L; public static final QueryGroup queryGroupOne = builder().name(NAME_ONE) ._id(_ID_ONE) - .mode(MONITOR_STRING) - .resourceLimits(Map.of(fromName(MEMORY_STRING), 0.3)) + .mutableQueryGroupFragment( + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.3)) + ) .updatedAt(TIMESTAMP_ONE) .build(); public static final QueryGroup queryGroupTwo = builder().name(NAME_TWO) ._id(_ID_TWO) - .mode(MONITOR_STRING) - .resourceLimits(Map.of(fromName(MEMORY_STRING), 0.6)) + .mutableQueryGroupFragment( + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.6)) + ) .updatedAt(TIMESTAMP_TWO) .build(); @@ -131,14 +132,35 @@ public static Tuple preparePersisten return new Tuple(queryGroupPersistenceService, clusterState); } - public static void assertEqualQueryGroups(Collection collectionOne, Collection collectionTwo) { + public static void assertEqualResourceLimits( + Map resourceLimitMapOne, + Map resourceLimitMapTwo + ) { + assertTrue(resourceLimitMapOne.keySet().containsAll(resourceLimitMapTwo.keySet())); + assertTrue(resourceLimitMapOne.values().containsAll(resourceLimitMapTwo.values())); + } + + public static void assertEqualQueryGroups( + Collection collectionOne, + Collection collectionTwo, + boolean assertUpdateAt + ) { assertEquals(collectionOne.size(), collectionTwo.size()); List listOne = new ArrayList<>(collectionOne); List listTwo = new ArrayList<>(collectionTwo); listOne.sort(Comparator.comparing(QueryGroup::getName)); listTwo.sort(Comparator.comparing(QueryGroup::getName)); for (int i = 0; i < listOne.size(); i++) { - assertTrue(listOne.get(i).equals(listTwo.get(i))); + if (assertUpdateAt) { + QueryGroup one = listOne.get(i); + QueryGroup two = listTwo.get(i); + assertEquals(one.getName(), two.getName()); + assertEquals(one.getResourceLimits(), two.getResourceLimits()); + assertEquals(one.getResiliencyMode(), two.getResiliencyMode()); + assertEquals(one.get_id(), two.get_id()); + } else { + assertEquals(listOne.get(i), listTwo.get(i)); + } } } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequestTests.java index b0fa96a46df80..dd9de4bf8fb1a 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequestTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequestTests.java @@ -35,6 +35,6 @@ public void testSerialization() throws IOException { List list2 = new ArrayList<>(); list1.add(queryGroupOne); list2.add(otherRequest.getQueryGroup()); - assertEqualQueryGroups(list1, list2); + assertEqualQueryGroups(list1, list2, false); } } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupResponseTests.java index ecb9a6b2dc0d2..3a2ce215d21b5 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/CreateQueryGroupResponseTests.java @@ -42,7 +42,7 @@ public void testSerialization() throws IOException { List listTwo = new ArrayList<>(); listOne.add(responseGroup); listTwo.add(otherResponseGroup); - QueryGroupTestUtils.assertEqualQueryGroups(listOne, listTwo); + QueryGroupTestUtils.assertEqualQueryGroups(listOne, listTwo, false); } /** @@ -56,10 +56,10 @@ public void testToXContentCreateQueryGroup() throws IOException { + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" + " \"name\" : \"query_group_one\",\n" + " \"resiliency_mode\" : \"monitor\",\n" - + " \"updated_at\" : 4513232413,\n" + " \"resource_limits\" : {\n" + " \"memory\" : 0.3\n" - + " }\n" + + " },\n" + + " \"updated_at\" : 4513232413\n" + "}"; assertEquals(expected, actual); } diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetQueryGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetQueryGroupResponseTests.java index 774f4b2d8db52..1a2ac282d86a4 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetQueryGroupResponseTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/GetQueryGroupResponseTests.java @@ -41,7 +41,7 @@ public void testSerializationSingleQueryGroup() throws IOException { GetQueryGroupResponse otherResponse = new GetQueryGroupResponse(streamInput); assertEquals(response.getRestStatus(), otherResponse.getRestStatus()); - QueryGroupTestUtils.assertEqualQueryGroups(response.getQueryGroups(), otherResponse.getQueryGroups()); + QueryGroupTestUtils.assertEqualQueryGroups(response.getQueryGroups(), otherResponse.getQueryGroups(), false); } /** @@ -58,7 +58,7 @@ public void testSerializationMultipleQueryGroup() throws IOException { GetQueryGroupResponse otherResponse = new GetQueryGroupResponse(streamInput); assertEquals(response.getRestStatus(), otherResponse.getRestStatus()); assertEquals(2, otherResponse.getQueryGroups().size()); - QueryGroupTestUtils.assertEqualQueryGroups(response.getQueryGroups(), otherResponse.getQueryGroups()); + QueryGroupTestUtils.assertEqualQueryGroups(response.getQueryGroups(), otherResponse.getQueryGroups(), false); } /** @@ -93,10 +93,10 @@ public void testToXContentGetSingleQueryGroup() throws IOException { + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" + " \"name\" : \"query_group_one\",\n" + " \"resiliency_mode\" : \"monitor\",\n" - + " \"updated_at\" : 4513232413,\n" + " \"resource_limits\" : {\n" + " \"memory\" : 0.3\n" - + " }\n" + + " },\n" + + " \"updated_at\" : 4513232413\n" + " }\n" + " ]\n" + "}"; @@ -119,19 +119,19 @@ public void testToXContentGetMultipleQueryGroup() throws IOException { + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" + " \"name\" : \"query_group_one\",\n" + " \"resiliency_mode\" : \"monitor\",\n" - + " \"updated_at\" : 4513232413,\n" + " \"resource_limits\" : {\n" + " \"memory\" : 0.3\n" - + " }\n" + + " },\n" + + " \"updated_at\" : 4513232413\n" + " },\n" + " {\n" + " \"_id\" : \"G5iIqHy4g7eK1qIAAAAIH53=1\",\n" + " \"name\" : \"query_group_two\",\n" + " \"resiliency_mode\" : \"monitor\",\n" - + " \"updated_at\" : 4513232415,\n" + " \"resource_limits\" : {\n" + " \"memory\" : 0.6\n" - + " }\n" + + " },\n" + + " \"updated_at\" : 4513232415\n" + " }\n" + " ]\n" + "}"; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/QueryGroupActionTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/QueryGroupActionTestUtils.java new file mode 100644 index 0000000000000..08d128ca7ed59 --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/QueryGroupActionTestUtils.java @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.wlm.MutableQueryGroupFragment; + +public class QueryGroupActionTestUtils { + public static UpdateQueryGroupRequest updateQueryGroupRequest(String name, MutableQueryGroupFragment mutableQueryGroupFragment) { + return new UpdateQueryGroupRequest(name, mutableQueryGroupFragment); + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequestTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequestTests.java new file mode 100644 index 0000000000000..b99f079e81984 --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequestTests.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; +import org.opensearch.wlm.ResourceType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.plugin.wlm.QueryGroupTestUtils.NAME_ONE; +import static org.opensearch.plugin.wlm.QueryGroupTestUtils.queryGroupOne; + +public class UpdateQueryGroupRequestTests extends OpenSearchTestCase { + + /** + * Test case to verify the serialization and deserialization of UpdateQueryGroupRequest. + */ + public void testSerialization() throws IOException { + UpdateQueryGroupRequest request = new UpdateQueryGroupRequest(NAME_ONE, queryGroupOne.getMutableQueryGroupFragment()); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + UpdateQueryGroupRequest otherRequest = new UpdateQueryGroupRequest(streamInput); + assertEquals(request.getName(), otherRequest.getName()); + assertEquals(request.getmMutableQueryGroupFragment(), otherRequest.getmMutableQueryGroupFragment()); + } + + /** + * Test case to verify the serialization and deserialization of UpdateQueryGroupRequest with only name field. + */ + public void testSerializationOnlyName() throws IOException { + UpdateQueryGroupRequest request = new UpdateQueryGroupRequest(NAME_ONE, new MutableQueryGroupFragment(null, new HashMap<>())); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + UpdateQueryGroupRequest otherRequest = new UpdateQueryGroupRequest(streamInput); + assertEquals(request.getName(), otherRequest.getName()); + assertEquals(request.getmMutableQueryGroupFragment(), otherRequest.getmMutableQueryGroupFragment()); + } + + /** + * Test case to verify the serialization and deserialization of UpdateQueryGroupRequest with only resourceLimits field. + */ + public void testSerializationOnlyResourceLimit() throws IOException { + UpdateQueryGroupRequest request = new UpdateQueryGroupRequest( + NAME_ONE, + new MutableQueryGroupFragment(null, Map.of(ResourceType.MEMORY, 0.4)) + ); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + UpdateQueryGroupRequest otherRequest = new UpdateQueryGroupRequest(streamInput); + assertEquals(request.getName(), otherRequest.getName()); + assertEquals(request.getmMutableQueryGroupFragment(), otherRequest.getmMutableQueryGroupFragment()); + } + + /** + * Tests invalid ResourceType. + */ + public void testInvalidResourceLimitList() { + assertThrows( + IllegalArgumentException.class, + () -> new UpdateQueryGroupRequest( + NAME_ONE, + new MutableQueryGroupFragment( + ResiliencyMode.MONITOR, + Map.of(ResourceType.MEMORY, 0.3, ResourceType.fromName("random"), 0.4) + ) + ) + ); + } + + /** + * Tests invalid resiliencyMode. + */ + public void testInvalidEnforcement() { + assertThrows( + IllegalArgumentException.class, + () -> new UpdateQueryGroupRequest( + NAME_ONE, + new MutableQueryGroupFragment(ResiliencyMode.fromName("random"), Map.of(ResourceType.fromName("memory"), 0.3)) + ) + ); + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponseTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponseTests.java new file mode 100644 index 0000000000000..a7ab4c6a682ef --- /dev/null +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupResponseTests.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.action; + +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.plugin.wlm.QueryGroupTestUtils; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.plugin.wlm.QueryGroupTestUtils.queryGroupOne; +import static org.mockito.Mockito.mock; + +public class UpdateQueryGroupResponseTests extends OpenSearchTestCase { + + /** + * Test case to verify the serialization and deserialization of UpdateQueryGroupResponse. + */ + public void testSerialization() throws IOException { + UpdateQueryGroupResponse response = new UpdateQueryGroupResponse(queryGroupOne, RestStatus.OK); + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + UpdateQueryGroupResponse otherResponse = new UpdateQueryGroupResponse(streamInput); + assertEquals(response.getRestStatus(), otherResponse.getRestStatus()); + QueryGroup responseGroup = response.getQueryGroup(); + QueryGroup otherResponseGroup = otherResponse.getQueryGroup(); + List list1 = new ArrayList<>(); + List list2 = new ArrayList<>(); + list1.add(responseGroup); + list2.add(otherResponseGroup); + QueryGroupTestUtils.assertEqualQueryGroups(list1, list2, false); + } + + /** + * Test case to verify the toXContent method of UpdateQueryGroupResponse. + */ + public void testToXContentUpdateSingleQueryGroup() throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + UpdateQueryGroupResponse otherResponse = new UpdateQueryGroupResponse(queryGroupOne, RestStatus.OK); + String actual = otherResponse.toXContent(builder, mock(ToXContent.Params.class)).toString(); + String expected = "{\n" + + " \"_id\" : \"AgfUO5Ja9yfsYlONlYi3TQ==\",\n" + + " \"name\" : \"query_group_one\",\n" + + " \"resiliency_mode\" : \"monitor\",\n" + + " \"resource_limits\" : {\n" + + " \"memory\" : 0.3\n" + + " },\n" + + " \"updated_at\" : 4513232413\n" + + "}"; + assertEquals(expected, actual); + } +} diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java index 5cb3d8fc6d11f..08b51fd46cfcf 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java @@ -24,8 +24,12 @@ import org.opensearch.plugin.wlm.QueryGroupTestUtils; import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse; import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupRequest; +import org.opensearch.plugin.wlm.action.UpdateQueryGroupResponse; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; import org.opensearch.wlm.ResourceType; import java.util.ArrayList; @@ -33,14 +37,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.mockito.ArgumentCaptor; import static org.opensearch.cluster.metadata.QueryGroup.builder; -import static org.opensearch.plugin.wlm.QueryGroupTestUtils.MEMORY_STRING; -import static org.opensearch.plugin.wlm.QueryGroupTestUtils.MONITOR_STRING; import static org.opensearch.plugin.wlm.QueryGroupTestUtils.NAME_NONE_EXISTED; import static org.opensearch.plugin.wlm.QueryGroupTestUtils.NAME_ONE; import static org.opensearch.plugin.wlm.QueryGroupTestUtils.NAME_TWO; @@ -55,6 +58,7 @@ import static org.opensearch.plugin.wlm.QueryGroupTestUtils.queryGroupOne; import static org.opensearch.plugin.wlm.QueryGroupTestUtils.queryGroupPersistenceService; import static org.opensearch.plugin.wlm.QueryGroupTestUtils.queryGroupTwo; +import static org.opensearch.plugin.wlm.action.QueryGroupActionTestUtils.updateQueryGroupRequest; import static org.opensearch.plugin.wlm.service.QueryGroupPersistenceService.QUERY_GROUP_COUNT_SETTING_NAME; import static org.opensearch.plugin.wlm.service.QueryGroupPersistenceService.SOURCE; import static org.mockito.ArgumentMatchers.any; @@ -83,7 +87,7 @@ public void testCreateQueryGroup() { List listTwo = new ArrayList<>(); listOne.add(queryGroupOne); listTwo.add(updatedGroupsMap.get(_ID_ONE)); - assertEqualQueryGroups(listOne, listTwo); + assertEqualQueryGroups(listOne, listTwo, false); } /** @@ -99,7 +103,7 @@ public void testCreateAnotherQueryGroup() { assertEquals(2, updatedGroups.size()); assertTrue(updatedGroups.containsKey(_ID_TWO)); Collection values = updatedGroups.values(); - assertEqualQueryGroups(queryGroupList(), new ArrayList<>(values)); + assertEqualQueryGroups(queryGroupList(), new ArrayList<>(values), false); } /** @@ -111,8 +115,7 @@ public void testCreateQueryGroupDuplicateName() { ClusterState clusterState = setup.v2(); QueryGroup toCreate = builder().name(NAME_ONE) ._id("W5iIqHyhgi4K1qIAAAAIHw==") - .mode(MONITOR_STRING) - .resourceLimits(Map.of(ResourceType.fromName(MEMORY_STRING), 0.3)) + .mutableQueryGroupFragment(new MutableQueryGroupFragment(ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.3))) .updatedAt(1690934400000L) .build(); assertThrows(RuntimeException.class, () -> queryGroupPersistenceService1.saveQueryGroupInClusterState(toCreate, clusterState)); @@ -126,8 +129,7 @@ public void testCreateQueryGroupOverflowAllocation() { Tuple setup = preparePersistenceServiceSetup(Map.of(_ID_TWO, queryGroupTwo)); QueryGroup toCreate = builder().name(NAME_ONE) ._id("W5iIqHyhgi4K1qIAAAAIHw==") - .mode(MONITOR_STRING) - .resourceLimits(Map.of(ResourceType.fromName(MEMORY_STRING), 0.41)) + .mutableQueryGroupFragment(new MutableQueryGroupFragment(ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.41))) .updatedAt(1690934400000L) .build(); @@ -143,8 +145,7 @@ public void testCreateQueryGroupOverflowAllocation() { public void testCreateQueryGroupOverflowCount() { QueryGroup toCreate = builder().name(NAME_NONE_EXISTED) ._id("W5iIqHyhgi4K1qIAAAAIHw==") - .mode(MONITOR_STRING) - .resourceLimits(Map.of(ResourceType.fromName(MEMORY_STRING), 0.5)) + .mutableQueryGroupFragment(new MutableQueryGroupFragment(ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.5))) .updatedAt(1690934400000L) .build(); Metadata metadata = Metadata.builder().queryGroups(Map.of(_ID_ONE, queryGroupOne, _ID_TWO, queryGroupTwo)).build(); @@ -267,7 +268,7 @@ public void testGetSingleQueryGroup() { List listTwo = new ArrayList<>(); listOne.add(QueryGroupTestUtils.queryGroupOne); listTwo.add(queryGroup); - QueryGroupTestUtils.assertEqualQueryGroups(listOne, listTwo); + QueryGroupTestUtils.assertEqualQueryGroups(listOne, listTwo, false); } /** @@ -281,7 +282,7 @@ public void testGetAllQueryGroups() { Set currentNAME = res.stream().map(QueryGroup::getName).collect(Collectors.toSet()); assertTrue(currentNAME.contains(QueryGroupTestUtils.NAME_ONE)); assertTrue(currentNAME.contains(QueryGroupTestUtils.NAME_TWO)); - QueryGroupTestUtils.assertEqualQueryGroups(QueryGroupTestUtils.queryGroupList(), res); + QueryGroupTestUtils.assertEqualQueryGroups(QueryGroupTestUtils.queryGroupList(), res, false); } /** @@ -316,7 +317,7 @@ public void testDeleteSingleQueryGroup() { assertEquals(1, afterDeletionGroups.size()); List oldQueryGroups = new ArrayList<>(); oldQueryGroups.add(queryGroupOne); - assertEqualQueryGroups(new ArrayList<>(afterDeletionGroups.values()), oldQueryGroups); + assertEqualQueryGroups(new ArrayList<>(afterDeletionGroups.values()), oldQueryGroups, false); } /** @@ -356,4 +357,159 @@ public void testDeleteInClusterStateMetadata() throws Exception { queryGroupPersistenceService.deleteInClusterStateMetadata(request, listener); verify(clusterService).submitStateUpdateTask(eq(SOURCE), any(AckedClusterStateUpdateTask.class)); } + + /** + * Tests updating a QueryGroup with all fields + */ + public void testUpdateQueryGroupAllFields() { + QueryGroup updated = builder().name(NAME_ONE) + ._id(_ID_ONE) + .mutableQueryGroupFragment(new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(ResourceType.MEMORY, 0.15))) + .updatedAt(1690934400000L) + .build(); + UpdateQueryGroupRequest updateQueryGroupRequest = updateQueryGroupRequest(NAME_ONE, updated.getMutableQueryGroupFragment()); + ClusterState newClusterState = queryGroupPersistenceService().updateQueryGroupInClusterState( + updateQueryGroupRequest, + clusterState() + ); + List updatedQueryGroups = new ArrayList<>(newClusterState.getMetadata().queryGroups().values()); + assertEquals(2, updatedQueryGroups.size()); + List expectedList = new ArrayList<>(); + expectedList.add(queryGroupTwo); + expectedList.add(updated); + assertEqualQueryGroups(expectedList, updatedQueryGroups, true); + } + + /** + * Tests updating a QueryGroup with only updated resourceLimits + */ + public void testUpdateQueryGroupResourceLimitsOnly() { + QueryGroup updated = builder().name(NAME_ONE) + ._id(_ID_ONE) + .mutableQueryGroupFragment(new MutableQueryGroupFragment(ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.15))) + .updatedAt(1690934400000L) + .build(); + UpdateQueryGroupRequest updateQueryGroupRequest = updateQueryGroupRequest(NAME_ONE, updated.getMutableQueryGroupFragment()); + ClusterState newClusterState = queryGroupPersistenceService().updateQueryGroupInClusterState( + updateQueryGroupRequest, + clusterState() + ); + List updatedQueryGroups = new ArrayList<>(newClusterState.getMetadata().queryGroups().values()); + assertEquals(2, updatedQueryGroups.size()); + Optional findUpdatedGroupOne = newClusterState.metadata() + .queryGroups() + .values() + .stream() + .filter(group -> group.getName().equals(NAME_ONE)) + .findFirst(); + Optional findUpdatedGroupTwo = newClusterState.metadata() + .queryGroups() + .values() + .stream() + .filter(group -> group.getName().equals(NAME_TWO)) + .findFirst(); + assertTrue(findUpdatedGroupOne.isPresent()); + assertTrue(findUpdatedGroupTwo.isPresent()); + List list1 = new ArrayList<>(); + list1.add(updated); + List list2 = new ArrayList<>(); + list2.add(findUpdatedGroupOne.get()); + assertEqualQueryGroups(list1, list2, true); + } + + /** + * Tests updating a QueryGroup with invalid name + */ + public void testUpdateQueryGroupNonExistedName() { + QueryGroupPersistenceService queryGroupPersistenceService = queryGroupPersistenceService(); + UpdateQueryGroupRequest updateQueryGroupRequest = updateQueryGroupRequest( + NAME_NONE_EXISTED, + new MutableQueryGroupFragment(ResiliencyMode.MONITOR, Map.of(ResourceType.MEMORY, 0.15)) + ); + assertThrows( + RuntimeException.class, + () -> queryGroupPersistenceService.updateQueryGroupInClusterState(updateQueryGroupRequest, clusterState()) + ); + List updatedQueryGroups = new ArrayList<>( + queryGroupPersistenceService.getClusterService().state().metadata().queryGroups().values() + ); + assertEquals(2, updatedQueryGroups.size()); + List expectedList = new ArrayList<>(); + expectedList.add(queryGroupTwo); + expectedList.add(queryGroupOne); + assertEqualQueryGroups(expectedList, updatedQueryGroups, true); + } + + /** + * Tests UpdateInClusterStateMetadata function + */ + public void testUpdateInClusterStateMetadata() { + ClusterService clusterService = mock(ClusterService.class); + @SuppressWarnings("unchecked") + ActionListener listener = mock(ActionListener.class); + QueryGroupPersistenceService queryGroupPersistenceService = new QueryGroupPersistenceService( + clusterService, + QueryGroupTestUtils.settings(), + clusterSettings() + ); + queryGroupPersistenceService.updateInClusterStateMetadata(null, listener); + verify(clusterService).submitStateUpdateTask(eq(SOURCE), any()); + } + + /** + * Tests UpdateInClusterStateMetadata function with inner functions + */ + public void testUpdateInClusterStateMetadataInner() { + ClusterService clusterService = mock(ClusterService.class); + @SuppressWarnings("unchecked") + ActionListener listener = mock(ActionListener.class); + QueryGroupPersistenceService queryGroupPersistenceService = new QueryGroupPersistenceService( + clusterService, + QueryGroupTestUtils.settings(), + clusterSettings() + ); + UpdateQueryGroupRequest updateQueryGroupRequest = updateQueryGroupRequest( + NAME_TWO, + new MutableQueryGroupFragment(ResiliencyMode.SOFT, new HashMap<>()) + ); + ArgumentCaptor captor = ArgumentCaptor.forClass(ClusterStateUpdateTask.class); + queryGroupPersistenceService.updateInClusterStateMetadata(updateQueryGroupRequest, listener); + verify(clusterService, times(1)).submitStateUpdateTask(eq(SOURCE), captor.capture()); + ClusterStateUpdateTask capturedTask = captor.getValue(); + assertEquals(queryGroupPersistenceService.updateQueryGroupThrottlingKey, capturedTask.getClusterManagerThrottlingKey()); + + doAnswer(invocation -> { + ClusterStateUpdateTask task = invocation.getArgument(1); + task.clusterStateProcessed(SOURCE, clusterState(), clusterState()); + return null; + }).when(clusterService).submitStateUpdateTask(anyString(), any()); + queryGroupPersistenceService.updateInClusterStateMetadata(updateQueryGroupRequest, listener); + verify(listener).onResponse(any(UpdateQueryGroupResponse.class)); + } + + /** + * Tests UpdateInClusterStateMetadata function with failure + */ + public void testUpdateInClusterStateMetadataFailure() { + ClusterService clusterService = mock(ClusterService.class); + @SuppressWarnings("unchecked") + ActionListener listener = mock(ActionListener.class); + QueryGroupPersistenceService queryGroupPersistenceService = new QueryGroupPersistenceService( + clusterService, + QueryGroupTestUtils.settings(), + clusterSettings() + ); + UpdateQueryGroupRequest updateQueryGroupRequest = updateQueryGroupRequest( + NAME_TWO, + new MutableQueryGroupFragment(ResiliencyMode.SOFT, new HashMap<>()) + ); + doAnswer(invocation -> { + ClusterStateUpdateTask task = invocation.getArgument(1); + Exception exception = new RuntimeException("Test Exception"); + task.onFailure(SOURCE, exception); + return null; + }).when(clusterService).submitStateUpdateTask(anyString(), any()); + queryGroupPersistenceService.updateInClusterStateMetadata(updateQueryGroupRequest, listener); + verify(listener).onFailure(any(RuntimeException.class)); + } } diff --git a/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/api/update_query_group_context.json b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/api/update_query_group_context.json new file mode 100644 index 0000000000000..fbfa2dde292ee --- /dev/null +++ b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/api/update_query_group_context.json @@ -0,0 +1,23 @@ +{ + "update_query_group_context": { + "stability": "experimental", + "url": { + "paths": [ + { + "path": "/_wlm/query_group/{name}", + "methods": ["PUT", "POST"], + "parts": { + "name": { + "type": "string", + "description": "QueryGroup name" + } + } + } + ] + }, + "params":{}, + "body":{ + "description":"The updated QueryGroup schema" + } + } +} diff --git a/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_query_group.yml b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_query_group.yml index a00314986a5cf..40ec665351094 100644 --- a/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_query_group.yml +++ b/plugins/workload-management/src/yamlRestTest/resources/rest-api-spec/test/wlm/10_query_group.yml @@ -29,6 +29,48 @@ - match: { query_groups.0.resource_limits.cpu: 0.4 } - match: { query_groups.0.resource_limits.memory: 0.2 } + - do: + update_query_group_context: + name: "analytics" + body: + { + "resiliency_mode": "monitor", + "resource_limits": { + "cpu": 0.42, + "memory": 0.22 + } + } + + - match: { name: "analytics" } + - match: { resiliency_mode: "monitor" } + - match: { resource_limits.cpu: 0.42 } + - match: { resource_limits.memory: 0.22 } + + - do: + catch: /resource_not_found_exception/ + update_query_group_context: + name: "analytics5" + body: + { + "resiliency_mode": "monitor", + "resource_limits": { + "cpu": 0.42, + "memory": 0.22 + } + } + + - do: + catch: /illegal_argument_exception/ + update_query_group_context: + name: "analytics" + body: + { + "resiliency_mode": "monitor", + "resource_limits": { + "cpu": 1.1 + } + } + - do: catch: /illegal_argument_exception/ create_query_group_context: diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java index a971aa58940ba..dcd96dceb4bf1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java @@ -17,6 +17,8 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; import org.opensearch.wlm.ResourceType; import org.joda.time.Instant; @@ -24,13 +26,15 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * Class to define the QueryGroup schema * { * "_id": "fafjafjkaf9ag8a9ga9g7ag0aagaga", * "resource_limits": { - * "memory": 0.4 + * "memory": 0.4, + * "cpu": 0.2 * }, * "resiliency_mode": "enforced", * "name": "analytics", @@ -42,44 +46,39 @@ public class QueryGroup extends AbstractDiffable implements ToXConte public static final String _ID_STRING = "_id"; public static final String NAME_STRING = "name"; - public static final String RESILIENCY_MODE_STRING = "resiliency_mode"; public static final String UPDATED_AT_STRING = "updated_at"; - public static final String RESOURCE_LIMITS_STRING = "resource_limits"; private static final int MAX_CHARS_ALLOWED_IN_NAME = 50; private final String name; private final String _id; - private final ResiliencyMode resiliencyMode; // It is an epoch in millis private final long updatedAtInMillis; - private final Map resourceLimits; + private final MutableQueryGroupFragment mutableQueryGroupFragment; - public QueryGroup(String name, ResiliencyMode resiliencyMode, Map resourceLimits) { - this(name, UUIDs.randomBase64UUID(), resiliencyMode, resourceLimits, Instant.now().getMillis()); + public QueryGroup(String name, MutableQueryGroupFragment mutableQueryGroupFragment) { + this(name, UUIDs.randomBase64UUID(), mutableQueryGroupFragment, Instant.now().getMillis()); } - public QueryGroup(String name, String _id, ResiliencyMode resiliencyMode, Map resourceLimits, long updatedAt) { + public QueryGroup(String name, String _id, MutableQueryGroupFragment mutableQueryGroupFragment, long updatedAt) { Objects.requireNonNull(name, "QueryGroup.name can't be null"); - Objects.requireNonNull(resourceLimits, "QueryGroup.resourceLimits can't be null"); - Objects.requireNonNull(resiliencyMode, "QueryGroup.resiliencyMode can't be null"); + Objects.requireNonNull(mutableQueryGroupFragment.getResourceLimits(), "QueryGroup.resourceLimits can't be null"); + Objects.requireNonNull(mutableQueryGroupFragment.getResiliencyMode(), "QueryGroup.resiliencyMode can't be null"); Objects.requireNonNull(_id, "QueryGroup._id can't be null"); validateName(name); - if (resourceLimits.isEmpty()) { + if (mutableQueryGroupFragment.getResourceLimits().isEmpty()) { throw new IllegalArgumentException("QueryGroup.resourceLimits should at least have 1 resource limit"); } - validateResourceLimits(resourceLimits); if (!isValid(updatedAt)) { throw new IllegalArgumentException("QueryGroup.updatedAtInMillis is not a valid epoch"); } this.name = name; this._id = _id; - this.resiliencyMode = resiliencyMode; - this.resourceLimits = resourceLimits; + this.mutableQueryGroupFragment = mutableQueryGroupFragment; this.updatedAtInMillis = updatedAt; } - private static boolean isValid(long updatedAt) { + public static boolean isValid(long updatedAt) { long minValidTimestamp = Instant.ofEpochMilli(0L).getMillis(); // Use Instant.now() to get the current time in seconds since epoch @@ -90,12 +89,22 @@ private static boolean isValid(long updatedAt) { } public QueryGroup(StreamInput in) throws IOException { - this( - in.readString(), - in.readString(), - ResiliencyMode.fromName(in.readString()), - in.readMap((i) -> ResourceType.fromName(i.readString()), StreamInput::readDouble), - in.readLong() + this(in.readString(), in.readString(), new MutableQueryGroupFragment(in), in.readLong()); + } + + public static QueryGroup updateExistingQueryGroup(QueryGroup existingGroup, MutableQueryGroupFragment mutableQueryGroupFragment) { + final Map updatedResourceLimits = new HashMap<>(existingGroup.getResourceLimits()); + final Map mutableFragmentResourceLimits = mutableQueryGroupFragment.getResourceLimits(); + if (mutableFragmentResourceLimits != null && !mutableFragmentResourceLimits.isEmpty()) { + updatedResourceLimits.putAll(mutableFragmentResourceLimits); + } + final ResiliencyMode mode = Optional.ofNullable(mutableQueryGroupFragment.getResiliencyMode()) + .orElse(existingGroup.getResiliencyMode()); + return new QueryGroup( + existingGroup.getName(), + existingGroup.get_id(), + new MutableQueryGroupFragment(mode, updatedResourceLimits), + Instant.now().getMillis() ); } @@ -103,8 +112,7 @@ public QueryGroup(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(_id); - out.writeString(resiliencyMode.getName()); - out.writeMap(resourceLimits, ResourceType::writeTo, StreamOutput::writeDouble); + mutableQueryGroupFragment.writeTo(out); out.writeLong(updatedAtInMillis); } @@ -114,34 +122,15 @@ public static void validateName(String name) { } } - private void validateResourceLimits(Map resourceLimits) { - for (Map.Entry resource : resourceLimits.entrySet()) { - Double threshold = resource.getValue(); - Objects.requireNonNull(resource.getKey(), "resourceName can't be null"); - Objects.requireNonNull(threshold, "resource limit threshold for" + resource.getKey().getName() + " : can't be null"); - - if (Double.compare(threshold, 0.0) <= 0 || Double.compare(threshold, 1.0) > 0) { - throw new IllegalArgumentException("resource value should be greater than 0 and less or equal to 1.0"); - } - } - } - @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); builder.field(_ID_STRING, _id); builder.field(NAME_STRING, name); - builder.field(RESILIENCY_MODE_STRING, resiliencyMode.getName()); - builder.field(UPDATED_AT_STRING, updatedAtInMillis); - // write resource limits - builder.startObject(RESOURCE_LIMITS_STRING); - for (ResourceType resourceType : ResourceType.values()) { - if (resourceLimits.containsKey(resourceType)) { - builder.field(resourceType.getName(), resourceLimits.get(resourceType)); - } + for (String fieldName : MutableQueryGroupFragment.acceptedFieldNames) { + mutableQueryGroupFragment.writeField(builder, fieldName); } - builder.endObject(); - + builder.field(UPDATED_AT_STRING, updatedAtInMillis); builder.endObject(); return builder; } @@ -160,27 +149,30 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; QueryGroup that = (QueryGroup) o; return Objects.equals(name, that.name) - && Objects.equals(resiliencyMode, that.resiliencyMode) - && Objects.equals(resourceLimits, that.resourceLimits) + && Objects.equals(mutableQueryGroupFragment, that.mutableQueryGroupFragment) && Objects.equals(_id, that._id) && updatedAtInMillis == that.updatedAtInMillis; } @Override public int hashCode() { - return Objects.hash(name, resourceLimits, updatedAtInMillis, _id); + return Objects.hash(name, mutableQueryGroupFragment, updatedAtInMillis, _id); } public String getName() { return name; } + public MutableQueryGroupFragment getMutableQueryGroupFragment() { + return mutableQueryGroupFragment; + } + public ResiliencyMode getResiliencyMode() { - return resiliencyMode; + return getMutableQueryGroupFragment().getResiliencyMode(); } public Map getResourceLimits() { - return resourceLimits; + return getMutableQueryGroupFragment().getResourceLimits(); } public String get_id() { @@ -199,37 +191,6 @@ public static Builder builder() { return new Builder(); } - /** - * This enum models the different QueryGroup resiliency modes - * SOFT - means that this query group can consume more than query group resource limits if node is not in duress - * ENFORCED - means that it will never breach the assigned limits and will cancel as soon as the limits are breached - * MONITOR - it will not cause any cancellation but just log the eligible task cancellations - */ - @ExperimentalApi - public enum ResiliencyMode { - SOFT("soft"), - ENFORCED("enforced"), - MONITOR("monitor"); - - private final String name; - - ResiliencyMode(String mode) { - this.name = mode; - } - - public String getName() { - return name; - } - - public static ResiliencyMode fromName(String s) { - for (ResiliencyMode mode : values()) { - if (mode.getName().equalsIgnoreCase(s)) return mode; - - } - throw new IllegalArgumentException("Invalid value for QueryGroupMode: " + s); - } - } - /** * Builder class for {@link QueryGroup} */ @@ -237,9 +198,8 @@ public static ResiliencyMode fromName(String s) { public static class Builder { private String name; private String _id; - private ResiliencyMode resiliencyMode; + private MutableQueryGroupFragment mutableQueryGroupFragment; private long updatedAt; - private Map resourceLimits; private Builder() {} @@ -257,8 +217,7 @@ public static Builder fromXContent(XContentParser parser) throws IOException { } String fieldName = ""; - // Map to hold resources - final Map resourceLimits = new HashMap<>(); + MutableQueryGroupFragment mutableQueryGroupFragment1 = new MutableQueryGroupFragment(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { fieldName = parser.currentName(); @@ -267,32 +226,21 @@ public static Builder fromXContent(XContentParser parser) throws IOException { builder._id(parser.text()); } else if (fieldName.equals(NAME_STRING)) { builder.name(parser.text()); - } else if (fieldName.equals(RESILIENCY_MODE_STRING)) { - builder.mode(parser.text()); + } else if (MutableQueryGroupFragment.shouldParse(fieldName)) { + mutableQueryGroupFragment1.parseField(parser, fieldName); } else if (fieldName.equals(UPDATED_AT_STRING)) { builder.updatedAt(parser.longValue()); } else { throw new IllegalArgumentException(fieldName + " is not a valid field in QueryGroup"); } } else if (token == XContentParser.Token.START_OBJECT) { - - if (!fieldName.equals(RESOURCE_LIMITS_STRING)) { - throw new IllegalArgumentException( - "QueryGroup.resourceLimits is an object and expected token was { " + " but found " + token - ); - } - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - } else { - resourceLimits.put(ResourceType.fromName(fieldName), parser.doubleValue()); - } + if (!MutableQueryGroupFragment.shouldParse(fieldName)) { + throw new IllegalArgumentException(fieldName + " is not a valid object in QueryGroup"); } - + mutableQueryGroupFragment1.parseField(parser, fieldName); } } - return builder.resourceLimits(resourceLimits); + return builder.mutableQueryGroupFragment(mutableQueryGroupFragment1); } public Builder name(String name) { @@ -305,8 +253,8 @@ public Builder _id(String _id) { return this; } - public Builder mode(String mode) { - this.resiliencyMode = ResiliencyMode.fromName(mode); + public Builder mutableQueryGroupFragment(MutableQueryGroupFragment mutableQueryGroupFragment) { + this.mutableQueryGroupFragment = mutableQueryGroupFragment; return this; } @@ -315,13 +263,12 @@ public Builder updatedAt(long updatedAt) { return this; } - public Builder resourceLimits(Map resourceLimits) { - this.resourceLimits = resourceLimits; - return this; + public QueryGroup build() { + return new QueryGroup(name, _id, mutableQueryGroupFragment, updatedAt); } - public QueryGroup build() { - return new QueryGroup(name, _id, resiliencyMode, resourceLimits, updatedAt); + public MutableQueryGroupFragment getMutableQueryGroupFragment() { + return mutableQueryGroupFragment; } } } diff --git a/server/src/main/java/org/opensearch/wlm/MutableQueryGroupFragment.java b/server/src/main/java/org/opensearch/wlm/MutableQueryGroupFragment.java new file mode 100644 index 0000000000000..8ea240132fea2 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/MutableQueryGroupFragment.java @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm; + +import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; + +/** + * Class to hold the fields that can be updated in a QueryGroup. + */ +@ExperimentalApi +public class MutableQueryGroupFragment extends AbstractDiffable { + + public static final String RESILIENCY_MODE_STRING = "resiliency_mode"; + public static final String RESOURCE_LIMITS_STRING = "resource_limits"; + private ResiliencyMode resiliencyMode; + private Map resourceLimits; + + public static final List acceptedFieldNames = List.of(RESILIENCY_MODE_STRING, RESOURCE_LIMITS_STRING); + + public MutableQueryGroupFragment() {} + + public MutableQueryGroupFragment(ResiliencyMode resiliencyMode, Map resourceLimits) { + validateResourceLimits(resourceLimits); + this.resiliencyMode = resiliencyMode; + this.resourceLimits = resourceLimits; + } + + public MutableQueryGroupFragment(StreamInput in) throws IOException { + if (in.readBoolean()) { + resourceLimits = in.readMap((i) -> ResourceType.fromName(i.readString()), StreamInput::readDouble); + } else { + resourceLimits = new HashMap<>(); + } + String updatedResiliencyMode = in.readOptionalString(); + resiliencyMode = updatedResiliencyMode == null ? null : ResiliencyMode.fromName(updatedResiliencyMode); + } + + interface FieldParser { + T parseField(XContentParser parser) throws IOException; + } + + static class ResiliencyModeParser implements FieldParser { + public ResiliencyMode parseField(XContentParser parser) throws IOException { + return ResiliencyMode.fromName(parser.text()); + } + } + + static class ResourceLimitsParser implements FieldParser> { + public Map parseField(XContentParser parser) throws IOException { + String fieldName = ""; + XContentParser.Token token; + final Map resourceLimits = new HashMap<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + } else { + resourceLimits.put(ResourceType.fromName(fieldName), parser.doubleValue()); + } + } + return resourceLimits; + } + } + + static class FieldParserFactory { + static Optional> fieldParserFor(String fieldName) { + if (fieldName.equals(RESOURCE_LIMITS_STRING)) { + return Optional.of(new ResourceLimitsParser()); + } else if (fieldName.equals(RESILIENCY_MODE_STRING)) { + return Optional.of(new ResiliencyModeParser()); + } + return Optional.empty(); + } + } + + private final Map> toXContentMap = Map.of(RESILIENCY_MODE_STRING, (builder) -> { + try { + builder.field(RESILIENCY_MODE_STRING, resiliencyMode.getName()); + return null; + } catch (IOException e) { + throw new IllegalStateException("writing error encountered for the field " + RESILIENCY_MODE_STRING); + } + }, RESOURCE_LIMITS_STRING, (builder) -> { + try { + builder.startObject(RESOURCE_LIMITS_STRING); + for (ResourceType resourceType : ResourceType.values()) { + if (resourceLimits.containsKey(resourceType)) { + builder.field(resourceType.getName(), resourceLimits.get(resourceType)); + } + } + builder.endObject(); + return null; + } catch (IOException e) { + throw new IllegalStateException("writing error encountered for the field " + RESOURCE_LIMITS_STRING); + } + }); + + public static boolean shouldParse(String field) { + return FieldParserFactory.fieldParserFor(field).isPresent(); + } + + public void parseField(XContentParser parser, String field) { + FieldParserFactory.fieldParserFor(field).ifPresent(fieldParser -> { + try { + Object value = fieldParser.parseField(parser); + if (field.equals(RESILIENCY_MODE_STRING)) { + setResiliencyMode((ResiliencyMode) value); + } else if (field.equals(RESOURCE_LIMITS_STRING)) { + setResourceLimits((Map) value); + } + } catch (IOException e) { + throw new IllegalArgumentException("parsing error encountered for the field " + field); + } + }); + } + + public void writeField(XContentBuilder builder, String field) { + toXContentMap.get(field).apply(builder); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (resourceLimits == null || resourceLimits.isEmpty()) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeMap(resourceLimits, ResourceType::writeTo, StreamOutput::writeDouble); + } + out.writeOptionalString(resiliencyMode == null ? null : resiliencyMode.getName()); + } + + public static void validateResourceLimits(Map resourceLimits) { + if (resourceLimits == null) { + return; + } + for (Map.Entry resource : resourceLimits.entrySet()) { + Double threshold = resource.getValue(); + Objects.requireNonNull(resource.getKey(), "resourceName can't be null"); + Objects.requireNonNull(threshold, "resource limit threshold for" + resource.getKey().getName() + " : can't be null"); + + if (Double.compare(threshold, 0.0) <= 0 || Double.compare(threshold, 1.0) > 0) { + throw new IllegalArgumentException("resource value should be greater than 0 and less or equal to 1.0"); + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MutableQueryGroupFragment that = (MutableQueryGroupFragment) o; + return Objects.equals(resiliencyMode, that.resiliencyMode) && Objects.equals(resourceLimits, that.resourceLimits); + } + + @Override + public int hashCode() { + return Objects.hash(resiliencyMode, resourceLimits); + } + + public ResiliencyMode getResiliencyMode() { + return resiliencyMode; + } + + public Map getResourceLimits() { + return resourceLimits; + } + + /** + * This enum models the different QueryGroup resiliency modes + * SOFT - means that this query group can consume more than query group resource limits if node is not in duress + * ENFORCED - means that it will never breach the assigned limits and will cancel as soon as the limits are breached + * MONITOR - it will not cause any cancellation but just log the eligible task cancellations + */ + @ExperimentalApi + public enum ResiliencyMode { + SOFT("soft"), + ENFORCED("enforced"), + MONITOR("monitor"); + + private final String name; + + ResiliencyMode(String mode) { + this.name = mode; + } + + public String getName() { + return name; + } + + public static ResiliencyMode fromName(String s) { + for (ResiliencyMode mode : values()) { + if (mode.getName().equalsIgnoreCase(s)) return mode; + + } + throw new IllegalArgumentException("Invalid value for QueryGroupMode: " + s); + } + } + + public void setResiliencyMode(ResiliencyMode resiliencyMode) { + this.resiliencyMode = resiliencyMode; + } + + public void setResourceLimits(Map resourceLimits) { + validateResourceLimits(resourceLimits); + this.resourceLimits = resourceLimits; + } +} diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java index f5e667de73d93..3f8d231ffb91e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java @@ -15,6 +15,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractDiffableSerializationTestCase; +import org.opensearch.wlm.MutableQueryGroupFragment; import org.opensearch.wlm.ResourceType; import java.io.IOException; @@ -33,8 +34,7 @@ public void testToXContent() throws IOException { new QueryGroup( "test", "ajakgakg983r92_4242", - QueryGroup.ResiliencyMode.ENFORCED, - Map.of(ResourceType.MEMORY, 0.5), + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.MEMORY, 0.5)), updatedAt ) ) @@ -44,7 +44,7 @@ public void testToXContent() throws IOException { queryGroupMetadata.toXContent(builder, null); builder.endObject(); assertEquals( - "{\"ajakgakg983r92_4242\":{\"_id\":\"ajakgakg983r92_4242\",\"name\":\"test\",\"resiliency_mode\":\"enforced\",\"updated_at\":1720047207,\"resource_limits\":{\"memory\":0.5}}}", + "{\"ajakgakg983r92_4242\":{\"_id\":\"ajakgakg983r92_4242\",\"name\":\"test\",\"resiliency_mode\":\"enforced\",\"resource_limits\":{\"memory\":0.5},\"updated_at\":1720047207}}", builder.toString() ); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java index f4d3e5ceb1784..ce1b1270fc94e 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java @@ -15,6 +15,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; import org.opensearch.wlm.ResourceType; import org.joda.time.Instant; @@ -26,20 +28,16 @@ public class QueryGroupTests extends AbstractSerializingTestCase { - private static final List allowedModes = List.of( - QueryGroup.ResiliencyMode.SOFT, - QueryGroup.ResiliencyMode.ENFORCED, - QueryGroup.ResiliencyMode.MONITOR - ); + private static final List allowedModes = List.of(ResiliencyMode.SOFT, ResiliencyMode.ENFORCED, ResiliencyMode.MONITOR); static QueryGroup createRandomQueryGroup(String _id) { String name = randomAlphaOfLength(10); Map resourceLimit = new HashMap<>(); resourceLimit.put(ResourceType.MEMORY, randomDoubleBetween(0.0, 0.80, false)); - return new QueryGroup(name, _id, randomMode(), resourceLimit, Instant.now().getMillis()); + return new QueryGroup(name, _id, new MutableQueryGroupFragment(randomMode(), resourceLimit), Instant.now().getMillis()); } - private static QueryGroup.ResiliencyMode randomMode() { + private static ResiliencyMode randomMode() { return allowedModes.get(randomIntBetween(0, allowedModes.size() - 1)); } @@ -74,37 +72,60 @@ protected QueryGroup createTestInstance() { public void testNullName() { assertThrows( NullPointerException.class, - () -> new QueryGroup(null, "_id", randomMode(), Collections.emptyMap(), Instant.now().getMillis()) + () -> new QueryGroup( + null, + "_id", + new MutableQueryGroupFragment(randomMode(), Collections.emptyMap()), + Instant.now().getMillis() + ) ); } public void testNullId() { assertThrows( NullPointerException.class, - () -> new QueryGroup("Dummy", null, randomMode(), Collections.emptyMap(), Instant.now().getMillis()) + () -> new QueryGroup( + "Dummy", + null, + new MutableQueryGroupFragment(randomMode(), Collections.emptyMap()), + Instant.now().getMillis() + ) ); } public void testNullResourceLimits() { - assertThrows(NullPointerException.class, () -> new QueryGroup("analytics", "_id", randomMode(), null, Instant.now().getMillis())); + assertThrows( + NullPointerException.class, + () -> new QueryGroup("analytics", "_id", new MutableQueryGroupFragment(randomMode(), null), Instant.now().getMillis()) + ); } public void testEmptyResourceLimits() { assertThrows( IllegalArgumentException.class, - () -> new QueryGroup("analytics", "_id", randomMode(), Collections.emptyMap(), Instant.now().getMillis()) + () -> new QueryGroup( + "analytics", + "_id", + new MutableQueryGroupFragment(randomMode(), Collections.emptyMap()), + Instant.now().getMillis() + ) ); } public void testIllegalQueryGroupMode() { assertThrows( NullPointerException.class, - () -> new QueryGroup("analytics", "_id", null, Map.of(ResourceType.MEMORY, 0.4), Instant.now().getMillis()) + () -> new QueryGroup( + "analytics", + "_id", + new MutableQueryGroupFragment(null, Map.of(ResourceType.MEMORY, 0.4)), + Instant.now().getMillis() + ) ); } public void testQueryGroupInitiation() { - QueryGroup queryGroup = new QueryGroup("analytics", randomMode(), Map.of(ResourceType.MEMORY, 0.4)); + QueryGroup queryGroup = new QueryGroup("analytics", new MutableQueryGroupFragment(randomMode(), Map.of(ResourceType.MEMORY, 0.4))); assertNotNull(queryGroup.getName()); assertNotNull(queryGroup.get_id()); assertNotNull(queryGroup.getResourceLimits()); @@ -117,11 +138,11 @@ public void testQueryGroupInitiation() { public void testIllegalQueryGroupName() { assertThrows( NullPointerException.class, - () -> new QueryGroup("a".repeat(51), "_id", null, Map.of(ResourceType.MEMORY, 0.4), Instant.now().getMillis()) + () -> new QueryGroup("a".repeat(51), "_id", new MutableQueryGroupFragment(), Instant.now().getMillis()) ); assertThrows( NullPointerException.class, - () -> new QueryGroup("", "_id", null, Map.of(ResourceType.MEMORY, 0.4), Instant.now().getMillis()) + () -> new QueryGroup("", "_id", new MutableQueryGroupFragment(), Instant.now().getMillis()) ); } @@ -132,8 +153,7 @@ public void testInvalidResourceLimitWhenInvalidSystemResourceValueIsGiven() { () -> new QueryGroup( "analytics", "_id", - randomMode(), - Map.of(ResourceType.MEMORY, randomDoubleBetween(1.1, 1.8, false)), + new MutableQueryGroupFragment(randomMode(), Map.of(ResourceType.MEMORY, randomDoubleBetween(1.1, 1.8, false))), Instant.now().getMillis() ) ); @@ -143,8 +163,7 @@ public void testValidQueryGroup() { QueryGroup queryGroup = new QueryGroup( "analytics", "_id", - randomMode(), - Map.of(ResourceType.MEMORY, randomDoubleBetween(0.01, 0.8, false)), + new MutableQueryGroupFragment(randomMode(), Map.of(ResourceType.MEMORY, randomDoubleBetween(0.01, 0.8, false))), Instant.ofEpochMilli(1717187289).getMillis() ); @@ -163,8 +182,7 @@ public void testToXContent() throws IOException { QueryGroup queryGroup = new QueryGroup( "TestQueryGroup", queryGroupId, - QueryGroup.ResiliencyMode.ENFORCED, - Map.of(ResourceType.CPU, 0.30, ResourceType.MEMORY, 0.40), + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.30, ResourceType.MEMORY, 0.40)), currentTimeInMillis ); XContentBuilder builder = JsonXContent.contentBuilder(); @@ -172,9 +190,9 @@ public void testToXContent() throws IOException { assertEquals( "{\"_id\":\"" + queryGroupId - + "\",\"name\":\"TestQueryGroup\",\"resiliency_mode\":\"enforced\",\"updated_at\":" + + "\",\"name\":\"TestQueryGroup\",\"resiliency_mode\":\"enforced\",\"resource_limits\":{\"cpu\":0.3,\"memory\":0.4},\"updated_at\":" + currentTimeInMillis - + ",\"resource_limits\":{\"cpu\":0.3,\"memory\":0.4}}", + + "}", builder.toString() ); } diff --git a/server/src/test/java/org/opensearch/wlm/MutableQueryGroupFragmentTests.java b/server/src/test/java/org/opensearch/wlm/MutableQueryGroupFragmentTests.java new file mode 100644 index 0000000000000..cfe53ddbd2c14 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/MutableQueryGroupFragmentTests.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class MutableQueryGroupFragmentTests extends OpenSearchTestCase { + + public void testSerializationDeserialization() throws IOException { + Map resourceLimits = new HashMap<>(); + resourceLimits.put(ResourceType.CPU, 0.5); + resourceLimits.put(ResourceType.MEMORY, 0.75); + MutableQueryGroupFragment mutableQueryGroupFragment = new MutableQueryGroupFragment( + MutableQueryGroupFragment.ResiliencyMode.SOFT, + resourceLimits + ); + BytesStreamOutput out = new BytesStreamOutput(); + mutableQueryGroupFragment.writeTo(out); + StreamInput in = out.bytes().streamInput(); + MutableQueryGroupFragment deserializedGroup = new MutableQueryGroupFragment(in); + assertEquals(mutableQueryGroupFragment, deserializedGroup); + } + + public void testSerializationDeserializationWithNull() throws IOException { + MutableQueryGroupFragment mutableQueryGroupFragment = new MutableQueryGroupFragment(); + BytesStreamOutput out = new BytesStreamOutput(); + mutableQueryGroupFragment.writeTo(out); + StreamInput in = out.bytes().streamInput(); + MutableQueryGroupFragment deserializedGroup = new MutableQueryGroupFragment(in); + assertEquals(0, deserializedGroup.getResourceLimits().size()); + assertEquals(mutableQueryGroupFragment.getResiliencyMode(), deserializedGroup.getResiliencyMode()); + } + + public void testValidateResourceLimits() { + Map invalidLimits = new HashMap<>(); + invalidLimits.put(ResourceType.CPU, 1.5); + Exception exception = assertThrows(IllegalArgumentException.class, () -> { + MutableQueryGroupFragment.validateResourceLimits(invalidLimits); + }); + String expectedMessage = "resource value should be greater than 0 and less or equal to 1.0"; + String actualMessage = exception.getMessage(); + assertTrue(actualMessage.contains(expectedMessage)); + } + + public void testSetMethodsWithNullAndEmptyValues() { + MutableQueryGroupFragment queryGroup = new MutableQueryGroupFragment(); + queryGroup.setResiliencyMode(null); + assertNull(queryGroup.getResiliencyMode()); + queryGroup.setResourceLimits(null); + assertNull(queryGroup.getResourceLimits()); + queryGroup.setResourceLimits(new HashMap<>()); + assertEquals(0, queryGroup.getResourceLimits().size()); + } +} From 5bf34d21d8345befbe02181c9b6fd28dd360eb2b Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Wed, 4 Sep 2024 10:51:02 +0530 Subject: [PATCH 036/357] [Snapshot V2] Support pinned timestamp in delete flow (#15256) Signed-off-by: Anshu Agarwal --- .../snapshots/DeleteSnapshotITV2.java | 332 ++++++++++++++++++ .../TransportCleanupRepositoryAction.java | 9 + .../RemoteStorePinnedTimestampService.java | 2 + .../opensearch/repositories/Repository.java | 60 +++- .../repositories/RepositoryData.java | 5 + .../blobstore/BlobStoreRepository.java | 309 +++++++++++++++- .../snapshots/SnapshotsService.java | 84 ++++- .../blobstore/BlobStoreRepositoryTests.java | 20 +- 8 files changed, 790 insertions(+), 31 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java new file mode 100644 index 0000000000000..02b6ea47172c7 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java @@ -0,0 +1,332 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class DeleteSnapshotITV2 extends AbstractSnapshotIntegTestCase { + + private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + + public void testDeleteShallowCopyV2() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-create-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + String snapshotName2 = "test-create-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + + createIndex(indexName3, getRemoteStoreBackedIndexSettings()); + indexRandomDocs(indexName3, 10); + CreateSnapshotResponse createSnapshotResponse2 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .get(); + snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); + + assertAcked(client().admin().indices().prepareDelete(indexName1)); + Thread.sleep(100); + + AcknowledgedResponse deleteResponse = client().admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, snapshotName2) + .setSnapshots(snapshotName2) + .get(); + assertTrue(deleteResponse.isAcknowledged()); + + // test delete non-existent snapshot + assertThrows( + SnapshotMissingException.class, + () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() + ); + + } + + public void testDeleteShallowCopyV2MultipleSnapshots() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-create-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + String snapshotName2 = "test-create-snapshot2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + + createIndex(indexName3, getRemoteStoreBackedIndexSettings()); + indexRandomDocs(indexName3, 10); + + CreateSnapshotResponse createSnapshotResponse2 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .setWaitForCompletion(true) + .get(); + snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); + + AcknowledgedResponse deleteResponse = client().admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, snapshotName1, snapshotName2) + .setSnapshots(snapshotName2) + .get(); + assertTrue(deleteResponse.isAcknowledged()); + + // test delete non-existent snapshot + assertThrows( + SnapshotMissingException.class, + () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() + ); + + } + + public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); + settings = Settings.builder() + .put(settings) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNode(settings); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(2); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + clusterManagerName + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); + + String numShards = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_NUMBER_OF_SHARDS); + + logger.info("--> create two remote index shallow snapshots"); + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, "snap1") + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo(); + + indexRandomDocs(remoteStoreEnabledIndexName, 25); + + CreateSnapshotResponse createSnapshotResponse2 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, "snap2") + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo2 = createSnapshotResponse2.getSnapshotInfo(); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + assertThat(snapshotInfo2.snapshotId().getName(), equalTo("snap2")); + + // delete remote store index + assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); + + logger.info("--> delete snapshot 2"); + + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); + Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); + + // Get total segments remote store directory file count for deleted index and shard 0 + int segmentFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo2.snapshotId().getName()) + .get(); + assertAcked(deleteSnapshotResponse); + + Thread.sleep(5000); + + assertBusy(() -> { + try { + assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountBeforeDeletingSnapshot1)); + } catch (Exception e) {} + }, 30, TimeUnit.SECONDS); + int segmentFilesCountAfterDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); + + logger.info("--> delete snapshot 1"); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + // on snapshot deletion, remote store segment files should get cleaned up for deleted index - `remote-index-1` + deleteSnapshotResponse = clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName()) + .get(); + assertAcked(deleteSnapshotResponse); + + // Delete is async. Give time for it + assertBusy(() -> { + try { + assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountAfterDeletingSnapshot1)); + } catch (Exception e) {} + }, 60, TimeUnit.SECONDS); + } + + private Settings snapshotV2Settings(Path remoteStoreRepoPath) { + Settings settings = Settings.builder() + .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + return settings; + } + + protected Settings.Builder snapshotRepoSettingsForShallowV2(Path path) { + final Settings.Builder settings = Settings.builder(); + settings.put("location", path); + settings.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE); + settings.put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + return settings; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index 0bf40d0af01dc..54b6d7a914a9b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -52,6 +52,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.repositories.RepositoriesService; @@ -97,6 +98,8 @@ public final class TransportCleanupRepositoryAction extends TransportClusterMana private final RemoteStoreLockManagerFactory remoteStoreLockManagerFactory; + private final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory; + @Override protected String executor() { return ThreadPool.Names.SAME; @@ -124,6 +127,11 @@ public TransportCleanupRepositoryAction( ); this.repositoriesService = repositoriesService; this.snapshotsService = snapshotsService; + this.remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( + () -> repositoriesService, + threadPool, + remoteStoreSettings.getSegmentsPathFixedPrefix() + ); this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory( () -> repositoriesService, remoteStoreSettings.getSegmentsPathFixedPrefix() @@ -277,6 +285,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, ActionListener.wrap(result -> after(null, result), e -> after(e, null)) ) ) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 782ba5e9a6540..3a7734fc0538f 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.collect.Tuple; @@ -42,6 +43,7 @@ * * @opensearch.internal */ +@ExperimentalApi public class RemoteStorePinnedTimestampService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteStorePinnedTimestampService.class); private static Tuple> pinnedTimestampsSet = new Tuple<>(-1L, Set.of()); diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 4338c42e20b4b..d4520beb5b570 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -50,9 +50,11 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.Store; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; @@ -207,11 +209,59 @@ void deleteSnapshots( /** * Deletes snapshots and releases respective lock files from remote store repository. * - * @param snapshotIds snapshot ids - * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began - * @param repositoryMetaVersion version of the updated repository metadata to write - * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files - * @param listener completion listener + * @param snapshotIds snapshot ids + * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began + * @param repositoryMetaVersion version of the updated repository metadata to write + * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files + * @param remoteSegmentStoreDirectoryFactory RemoteSegmentStoreDirectoryFactory to be used for cleaning up remote store segment files + * @param remoteStorePinnedTimestampService service for pinning and unpinning of the timestamp + * @param snapshotIdsPinnedTimestampMap map of snapshots ids and the pinned timestamp + * @param isShallowSnapshotV2 true for shallow snapshots v2 + * @param listener completion listener + */ + default void deleteSnapshotsInternal( + Collection snapshotIds, + long repositoryStateId, + Version repositoryMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + Map snapshotIdsPinnedTimestampMap, + boolean isShallowSnapshotV2, + ActionListener listener + ) { + throw new UnsupportedOperationException(); + } + + /** + * Deletes snapshots and unpin the snapshot timestamp using remoteStorePinnedTimestampService + * + * @param snapshotsWithPinnedTimestamp map of snapshot ids and the pinned timestamps + * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began + * @param repositoryMetaVersion version of the updated repository metadata to write + * @param remoteSegmentStoreDirectoryFactory RemoteSegmentStoreDirectoryFactory to be used for cleaning up remote store segment files + * @param remoteStorePinnedTimestampService service for pinning and unpinning of the timestamp + * @param listener completion listener + */ + default void deleteSnapshotsWithPinnedTimestamp( + Map snapshotsWithPinnedTimestamp, + long repositoryStateId, + Version repositoryMetaVersion, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + ActionListener listener + ) { + throw new UnsupportedOperationException(); + } + + /** + * Deletes snapshots and releases respective lock files from remote store repository + * + * @param snapshotIds + * @param repositoryStateId + * @param repositoryMetaVersion + * @param remoteStoreLockManagerFactory + * @param listener */ default void deleteSnapshotsAndReleaseLockFiles( Collection snapshotIds, diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryData.java b/server/src/main/java/org/opensearch/repositories/RepositoryData.java index 09cb654604c06..7921a9589b443 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryData.java @@ -111,6 +111,11 @@ public final class RepositoryData { * The indices found in the repository across all snapshots, as a name to {@link IndexId} mapping */ private final Map indices; + + public Map> getIndexSnapshots() { + return indexSnapshots; + } + /** * The snapshots that each index belongs to. */ diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index aff48f717eb44..9ce9c0823429a 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -115,6 +115,7 @@ import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import org.opensearch.index.remote.RemoteStorePathStrategy.SnapshotShardPathInput; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; @@ -134,6 +135,7 @@ import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.IndexMetaDataGenerations; import org.opensearch.repositories.Repository; @@ -189,6 +191,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS; +import static org.opensearch.snapshots.SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER; /** * BlobStore - based implementation of Snapshot Repository @@ -969,11 +972,15 @@ public RepositoryStats stats() { return new RepositoryStats(store.stats()); } - public void deleteSnapshotsAndReleaseLockFiles( + public void deleteSnapshotsInternal( Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + Map snapshotIdsPinnedTimestampMap, + boolean isShallowSnapshotV2, ActionListener listener ) { if (isReadOnly()) { @@ -995,6 +1002,10 @@ protected void doRun() throws Exception { repositoryData, repositoryMetaVersion, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, + remoteStorePinnedTimestampService, + snapshotIdsPinnedTimestampMap, + isShallowSnapshotV2, listener ); } @@ -1007,6 +1018,49 @@ public void onFailure(Exception e) { } } + @Override + public void deleteSnapshotsWithPinnedTimestamp( + Map snapshotIdPinnedTimestampMap, + long repositoryStateId, + Version repositoryMetaVersion, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + ActionListener listener + ) { + deleteSnapshotsInternal( + snapshotIdPinnedTimestampMap.keySet(), + repositoryStateId, + repositoryMetaVersion, + null, // Passing null since no remote store lock files need to be cleaned up. + remoteSegmentStoreDirectoryFactory, + remoteStorePinnedTimestampService, + snapshotIdPinnedTimestampMap, + true, // true only for shallow snapshot v2 + listener + ); + } + + @Override + public void deleteSnapshotsAndReleaseLockFiles( + Collection snapshotIds, + long repositoryStateId, + Version repositoryMetaVersion, + RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + ActionListener listener + ) { + deleteSnapshotsInternal( + snapshotIds, + repositoryStateId, + repositoryMetaVersion, + remoteStoreLockManagerFactory, + null, + null, + Collections.emptyMap(), + false, + listener + ); + } + @Override public void deleteSnapshots( Collection snapshotIds, @@ -1014,11 +1068,15 @@ public void deleteSnapshots( Version repositoryMetaVersion, ActionListener listener ) { - deleteSnapshotsAndReleaseLockFiles( + deleteSnapshotsInternal( snapshotIds, repositoryStateId, repositoryMetaVersion, null, // Passing null since no remote store lock files need to be cleaned up. + null, // Passing null since no remote store segment files need to be cleaned up + null, + Collections.emptyMap(), + false, listener ); } @@ -1083,6 +1141,10 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map snapshotIdPinnedTimestampMap, + boolean isShallowSnapshotV2, ActionListener listener ) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets @@ -1128,7 +1194,23 @@ private void doDeleteShardSnapshots( ); }, listener::onFailure); // Once we have updated the repository, run the clean-ups + final StepListener pinnedTimestampListener = new StepListener<>(); writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> { + if (snapshotIdPinnedTimestampMap == null || snapshotIdPinnedTimestampMap.isEmpty()) { + pinnedTimestampListener.onResponse(updatedRepoData); + } else { + removeSnapshotsPinnedTimestamp( + snapshotIdPinnedTimestampMap, + this, + updatedRepoData, + remoteStorePinnedTimestampService, + pinnedTimestampListener + ); + } + }, listener::onFailure); + + pinnedTimestampListener.whenComplete(updatedRepoData -> { + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion final ActionListener afterCleanupsListener = new GroupedActionListener<>( ActionListener.wrap(() -> listener.onResponse(updatedRepoData)), @@ -1152,20 +1234,116 @@ private void doDeleteShardSnapshots( foundIndices, rootBlobs, updatedRepoData, + repositoryData, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, afterCleanupsListener, idToShardInfoMap ); - asyncCleanupUnlinkedShardLevelBlobs( - repositoryData, - snapshotIds, - writeShardMetaDataAndComputeDeletesStep.result(), - remoteStoreLockManagerFactory, - afterCleanupsListener - ); + if (isShallowSnapshotV2) { + cleanUpRemoteStoreFilesForDeletedIndicesV2( + repositoryData, + snapshotIds, + writeShardMetaDataAndComputeDeletesStep.result(), + remoteSegmentStoreDirectoryFactory, + afterCleanupsListener + ); + } else { + asyncCleanupUnlinkedShardLevelBlobs( + repositoryData, + snapshotIds, + writeShardMetaDataAndComputeDeletesStep.result(), + remoteStoreLockManagerFactory, + afterCleanupsListener + ); + } }, listener::onFailure); } + private void cleanUpRemoteStoreFilesForDeletedIndicesV2( + RepositoryData repositoryData, + Collection snapshotIds, + Collection result, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + ActionListener afterCleanupsListener + ) { + try { + Set uniqueIndexIds = new HashSet<>(); + for (ShardSnapshotMetaDeleteResult shardSnapshotMetaDeleteResult : result) { + uniqueIndexIds.add(shardSnapshotMetaDeleteResult.indexId.getId()); + } + // iterate through all the indices and trigger remote store directory cleanup for deleted index segments + for (String indexId : uniqueIndexIds) { + cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory); + } + afterCleanupsListener.onResponse(null); + } catch (Exception e) { + logger.warn("Exception during cleanup of remote directory files for snapshot v2", e); + afterCleanupsListener.onFailure(e); + } + + } + + private void removeSnapshotsPinnedTimestamp( + Map snapshotsWithPinnedTimestamp, + Repository repository, + RepositoryData repositoryData, + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + ActionListener pinnedTimestampListener + ) { + // Create a GroupedActionListener to aggregate the results of all unpin operations + GroupedActionListener groupedListener = new GroupedActionListener<>( + ActionListener.wrap( + // This is called once all operations have succeeded + ignored -> pinnedTimestampListener.onResponse(repositoryData), + // This is called if any operation fails + pinnedTimestampListener::onFailure + ), + snapshotsWithPinnedTimestamp.size() + ); + + snapshotsWithPinnedTimestamp.forEach((snapshotId, pinnedTimestamp) -> { + removeSnapshotPinnedTimestamp( + remoteStorePinnedTimestampService, + snapshotId, + repository.getMetadata().name(), + pinnedTimestamp, + groupedListener + ); + }); + } + + private void removeSnapshotPinnedTimestamp( + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService, + SnapshotId snapshotId, + String repository, + long timestampToUnpin, + ActionListener listener + ) { + remoteStorePinnedTimestampService.unpinTimestamp( + timestampToUnpin, + repository + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotId.getUUID(), + new ActionListener() { + @Override + public void onResponse(Void unused) { + logger.debug("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName()); + listener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + logger.error( + "Failed to unpin timestamp {} for snapshot {} with exception {}", + timestampToUnpin, + snapshotId.getName(), + e + ); + listener.onFailure(e); + } + } + ); + } + /** * Cleans up the indices and data corresponding to all it's shards. * @@ -1182,7 +1360,9 @@ private void cleanupUnlinkedRootAndIndicesBlobs( Map foundIndices, Map rootBlobs, RepositoryData updatedRepoData, + RepositoryData oldRepoData, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, ActionListener listener, Map idToShardInfoMap ) { @@ -1191,7 +1371,9 @@ private void cleanupUnlinkedRootAndIndicesBlobs( foundIndices, rootBlobs, updatedRepoData, + oldRepoData, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, ActionListener.map(listener, ignored -> null), idToShardInfoMap ); @@ -1571,7 +1753,9 @@ private void cleanupStaleBlobs( Map foundIndices, Map rootBlobs, RepositoryData newRepoData, + RepositoryData oldRepoData, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, ActionListener listener, Map idToShardInfoMap ) { @@ -1600,9 +1784,12 @@ private void cleanupStaleBlobs( } else { Map snapshotShardPaths = getSnapshotShardPaths(); cleanupStaleIndices( + deletedSnapshots, foundIndices, survivingIndexIds, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, + oldRepoData, groupedListener, snapshotShardPaths, idToShardInfoMap @@ -1632,12 +1819,14 @@ private Map getSnapshotShardPaths() { * @param repositoryStateId Current repository state id * @param repositoryMetaVersion version of the updated repository metadata to write * @param remoteStoreLockManagerFactory RemoteStoreLockManagerFactory to be used for cleaning up remote store lock files. + * @param remoteSegmentStoreDirectoryFactory RemoteSegmentStoreDirectoryFactory to be used for cleaning up remote store segments. * @param listener Listener to complete when done */ public void cleanup( long repositoryStateId, Version repositoryMetaVersion, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, ActionListener listener ) { try { @@ -1670,7 +1859,9 @@ public void cleanup( foundIndices, rootBlobs, repositoryData, + repositoryData, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, ActionListener.map(listener, RepositoryCleanupResult::new), Collections.emptyMap() ), @@ -1761,9 +1952,12 @@ private List cleanupStaleRootFiles( } void cleanupStaleIndices( + Collection deletedSnapshots, Map foundIndices, Set survivingIndexIds, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RepositoryData oldRepoData, GroupedActionListener listener, Map snapshotShardPaths, Map idToShardInfoMap @@ -1791,8 +1985,11 @@ void cleanupStaleIndices( ); for (int i = 0; i < workers; ++i) { executeOneStaleIndexDelete( + deletedSnapshots, staleIndicesToDelete, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, + oldRepoData, groupedListener, snapshotShardPaths, idToShardInfoMap @@ -1827,8 +2024,11 @@ private static boolean isIndexPresent(ClusterService clusterService, String inde * @throws InterruptedException if the thread is interrupted while waiting */ private void executeOneStaleIndexDelete( + Collection deletedSnapshots, BlockingQueue> staleIndicesToDelete, RemoteStoreLockManagerFactory remoteStoreLockManagerFactory, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + RepositoryData oldRepoData, GroupedActionListener listener, Map snapshotShardPaths, Map idToShardInfoMap @@ -1859,6 +2059,10 @@ private void executeOneStaleIndexDelete( // If there are matchingShardPaths, then we delete them after we have deleted the shard data. deleteResult = deleteResult.add(cleanUpStaleSnapshotShardPathsFile(matchingShardPaths, snapshotShardPaths)); + if (remoteSegmentStoreDirectoryFactory != null) { + cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory); + } + // Finally, we delete the [base_path]/indexId folder deleteResult = deleteResult.add(indexEntry.getValue().delete()); // Deleting the index folder logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); @@ -1880,8 +2084,11 @@ private void executeOneStaleIndexDelete( return DeleteResult.ZERO; } finally { executeOneStaleIndexDelete( + deletedSnapshots, staleIndicesToDelete, remoteStoreLockManagerFactory, + remoteSegmentStoreDirectoryFactory, + oldRepoData, listener, snapshotShardPaths, idToShardInfoMap @@ -1890,6 +2097,90 @@ private void executeOneStaleIndexDelete( })); } + /** + * Cleans up the remote store directory if needed. + *

This method cleans up segments in the remote store directory for deleted indices. + * This cleanup flow is executed only for v2 snapshots. For v1 snapshots, + * the cleanup is done per shard after releasing the lock files. + *

+ * + *

Since this method requires old repository data to fetch index metadata of the deleted index, + * the cleanup won't happen on retries in case of failures. This is because subsequent retries may + * not have access to the older repository data.

+ * + * @param indexSnId The snapshot index id of the index to be cleaned up + * @param oldRepoData The old repository metadata used to fetch the index metadata. + * @param remoteSegmentStoreDirectoryFactory RemoteSegmentStoreDirectoryFactory to be used for cleaning up remote + * store segments + */ + private void cleanRemoteStoreDirectoryIfNeeded( + Collection deletedSnapshots, + String indexSnId, + RepositoryData oldRepoData, + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory + ) { + assert (indexSnId != null); + + IndexId indexId = null; + List snapshotIds = Collections.emptyList(); + try { + for (Map.Entry> entry : oldRepoData.getIndexSnapshots().entrySet()) { + indexId = entry.getKey(); + if (indexId != null && indexId.getId().equals(indexSnId)) { + snapshotIds = entry.getValue(); + break; + } + } + if (snapshotIds.isEmpty()) { + logger.info("No snapshots found for indexSnId: {}", indexSnId); + return; + } + for (SnapshotId snapshotId : snapshotIds) { + try { + // skip cleanup for snapshot not present in deleted snapshots list + if (!deletedSnapshots.contains(snapshotId)) { + continue; + } + IndexMetadata prevIndexMetadata = this.getSnapshotIndexMetaData(oldRepoData, snapshotId, indexId); + if (prevIndexMetadata != null && !isIndexPresent(clusterService, prevIndexMetadata.getIndexUUID())) { + String remoteStoreRepository = IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get( + prevIndexMetadata.getSettings() + ); + assert (remoteStoreRepository != null); + + RemoteStorePathStrategy remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy( + prevIndexMetadata + ); + + for (int shardId = 0; shardId < prevIndexMetadata.getNumberOfShards(); shardId++) { + remoteDirectoryCleanupAsync( + remoteSegmentStoreDirectoryFactory, + threadPool, + remoteStoreRepository, + prevIndexMetadata.getIndexUUID(), + new ShardId(Index.UNKNOWN_INDEX_NAME, prevIndexMetadata.getIndexUUID(), shardId), + ThreadPool.Names.REMOTE_PURGE, + remoteStorePathStrategy + ); + } + } + } catch (Exception e) { + logger.warn( + new ParameterizedMessage( + "Exception during cleanup of remote directory for snapshot [{}] deleted index [{}]", + snapshotId, + indexSnId + ), + e + ); + } + } + } catch (Exception e) { + logger.error(new ParameterizedMessage("Exception during the remote directory cleanup for indecSnId [{}]", indexSnId), e); + } + + } + /** * Finds and returns a list of shard paths that match the given index ID. * diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 8b595fb7135c1..c4c77b352addb 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -91,6 +91,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; @@ -124,6 +125,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.function.Consumer; import java.util.function.Function; @@ -162,6 +164,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final RemoteStoreLockManagerFactory remoteStoreLockManagerFactory; + private final RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory; + private final ThreadPool threadPool; private final Map>>> snapshotCompletionListeners = @@ -205,7 +209,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus Setting.Property.Dynamic ); - private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__"; + public static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__"; /** * Setting to specify the maximum number of shards that can be included in the result for the snapshot status * API call. Note that it does not apply to V2-shallow snapshots. @@ -237,6 +241,11 @@ public SnapshotsService( remoteStoreSettings.getSegmentsPathFixedPrefix() ); this.threadPool = transportService.getThreadPool(); + this.remoteSegmentStoreDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( + () -> repositoriesService, + threadPool, + remoteStoreSettings.getSegmentsPathFixedPrefix() + ); this.transportService = transportService; this.remoteStorePinnedTimestampService = remoteStorePinnedTimestampService; @@ -2468,18 +2477,67 @@ private void deleteSnapshotsFromRepository( // the flag. This can be improved by having the info whether there ever were any shallow snapshot present in this repository // or not in RepositoryData. // SEE https://github.com/opensearch-project/OpenSearch/issues/8610 - final boolean cleanupRemoteStoreLockFiles = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings()); - if (cleanupRemoteStoreLockFiles) { - repository.deleteSnapshotsAndReleaseLockFiles( - snapshotIds, - repositoryData.getGenId(), - minCompatibleVersion(minNodeVersion, repositoryData, snapshotIds), - remoteStoreLockManagerFactory, - ActionListener.wrap(updatedRepoData -> { - logger.info("snapshots {} deleted", snapshotIds); - removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); - }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData)) - ); + final boolean remoteStoreShallowCopyEnabled = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings()); + if (remoteStoreShallowCopyEnabled) { + Map snapshotsWithPinnedTimestamp = new ConcurrentHashMap<>(); + List snapshotsWithLockFiles = Collections.synchronizedList(new ArrayList<>()); + + CountDownLatch latch = new CountDownLatch(1); + + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> { + try { + for (SnapshotId snapshotId : snapshotIds) { + try { + SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId); + if (snapshotInfo.getPinnedTimestamp() > 0) { + snapshotsWithPinnedTimestamp.put(snapshotId, snapshotInfo.getPinnedTimestamp()); + } else { + snapshotsWithLockFiles.add(snapshotId); + } + } catch (Exception e) { + logger.warn("Failed to get snapshot info for {} with exception {}", snapshotId, e); + removeSnapshotDeletionFromClusterState(deleteEntry, e, repositoryData); + } + } + } finally { + latch.countDown(); + } + }); + try { + latch.await(); + if (snapshotsWithLockFiles.size() > 0) { + repository.deleteSnapshotsAndReleaseLockFiles( + snapshotsWithLockFiles, + repositoryData.getGenId(), + minCompatibleVersion(minNodeVersion, repositoryData, snapshotsWithLockFiles), + remoteStoreLockManagerFactory, + ActionListener.wrap(updatedRepoData -> { + logger.info("snapshots {} deleted", snapshotsWithLockFiles); + removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); + }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData)) + ); + } + if (snapshotsWithPinnedTimestamp.size() > 0) { + + repository.deleteSnapshotsWithPinnedTimestamp( + snapshotsWithPinnedTimestamp, + repositoryData.getGenId(), + minCompatibleVersion(minNodeVersion, repositoryData, snapshotsWithPinnedTimestamp.keySet()), + remoteSegmentStoreDirectoryFactory, + remoteStorePinnedTimestampService, + ActionListener.wrap(updatedRepoData -> { + logger.info("snapshots {} deleted", snapshotsWithPinnedTimestamp); + removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); + }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData)) + ); + } + + } catch (InterruptedException e) { + logger.error("Interrupted while waiting for snapshot info processing", e); + Thread.currentThread().interrupt(); + removeSnapshotDeletionFromClusterState(deleteEntry, e, repositoryData); + } + } else { repository.deleteSnapshots( snapshotIds, diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 63257a5575970..d9622aae4c378 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -54,6 +54,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; import org.opensearch.indices.recovery.RecoverySettings; @@ -74,6 +75,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -459,11 +461,18 @@ public void testCleanupStaleIndices() throws Exception { foundIndices.put("stale-index", staleIndexContainer); foundIndices.put("current-index", currentIndexContainer); + List snapshotIds = new ArrayList<>(); + snapshotIds.add(new SnapshotId("snap1", UUIDs.randomBase64UUID())); + snapshotIds.add(new SnapshotId("snap2", UUIDs.randomBase64UUID())); + Set survivingIndexIds = new HashSet<>(); survivingIndexIds.add("current-index"); + RepositoryData repositoryData = generateRandomRepoData(); + // Create a mock RemoteStoreLockManagerFactory RemoteStoreLockManagerFactory mockRemoteStoreLockManagerFactory = mock(RemoteStoreLockManagerFactory.class); + RemoteSegmentStoreDirectoryFactory mockRemoteSegmentStoreDirectoryFactory = mock(RemoteSegmentStoreDirectoryFactory.class); RemoteStoreLockManager mockLockManager = mock(RemoteStoreLockManager.class); when(mockRemoteStoreLockManagerFactory.newLockManager(anyString(), anyString(), anyString(), any())).thenReturn(mockLockManager); @@ -479,9 +488,9 @@ public void testCleanupStaleIndices() throws Exception { // Mock the cleanupStaleIndices method to call our test implementation doAnswer(invocation -> { - Map indices = invocation.getArgument(0); - Set surviving = invocation.getArgument(1); - GroupedActionListener listener = invocation.getArgument(3); + Map indices = invocation.getArgument(1); + Set surviving = invocation.getArgument(2); + GroupedActionListener listener = invocation.getArgument(6); // Simulate the cleanup process DeleteResult result = DeleteResult.ZERO; @@ -494,7 +503,7 @@ public void testCleanupStaleIndices() throws Exception { listener.onResponse(result); return null; - }).when(repository).cleanupStaleIndices(any(), any(), any(), any(), any(), anyMap()); + }).when(repository).cleanupStaleIndices(any(), any(), any(), any(), any(), any(), any(), any(), anyMap()); AtomicReference> resultReference = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); @@ -509,9 +518,12 @@ public void testCleanupStaleIndices() throws Exception { // Call the method we're testing repository.cleanupStaleIndices( + snapshotIds, foundIndices, survivingIndexIds, mockRemoteStoreLockManagerFactory, + null, + repositoryData, listener, mockSnapshotShardPaths, Collections.emptyMap() From 17b5f98c5e28c5b56fe88d39513c4ce119600b9c Mon Sep 17 00:00:00 2001 From: Sumit Bansal Date: Wed, 4 Sep 2024 12:10:28 +0530 Subject: [PATCH 037/357] ClusterManagerTaskThrottler Improvements (#15508) * [ClusterManagerTaskThrottler Improvements] : Add shallow check in ClusterManagerTaskThrottler to fail fast before computeIfPresent to avoid lock when queue is full Signed-off-by: Sumit Bansal --- CHANGELOG.md | 1 + .../service/ClusterManagerTaskThrottler.java | 75 ++++--- .../ClusterManagerThrottlingException.java | 6 + .../ClusterManagerTaskThrottlerTests.java | 187 ++++++++++++++++-- 4 files changed, 234 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bf2c1ae2b8b..aa25dc88a64d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) +- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java index 827f3a12fbce4..39ce218dd801a 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerTaskThrottler.java @@ -33,7 +33,7 @@ *

* Set specific setting to for setting the threshold of throttling of particular task type. * e.g : Set "cluster_manager.throttling.thresholds.put_mapping" to set throttling limit of "put mapping" tasks, - * Set it to default value(-1) to disable the throttling for this task type. + * Set it to default value(-1) to disable the throttling for this task type. */ public class ClusterManagerTaskThrottler implements TaskBatcherListener { private static final Logger logger = LogManager.getLogger(ClusterManagerTaskThrottler.class); @@ -69,7 +69,7 @@ public class ClusterManagerTaskThrottler implements TaskBatcherListener { private final int MIN_THRESHOLD_VALUE = -1; // Disabled throttling private final ClusterManagerTaskThrottlerListener clusterManagerTaskThrottlerListener; - private final ConcurrentMap tasksCount; + final ConcurrentMap tasksCount; private final ConcurrentMap tasksThreshold; private final Supplier minNodeVersionSupplier; @@ -209,30 +209,59 @@ Long getThrottlingLimit(final String taskKey) { return tasksThreshold.get(taskKey); } + private void failFastWhenThrottlingThresholdsAreAlreadyBreached( + final boolean throttlingEnabledWithThreshold, + final Long threshold, + final long existingTaskCount, + final int incomingTaskCount, + final String taskThrottlingKey + ) { + if (throttlingEnabledWithThreshold && shouldThrottle(threshold, existingTaskCount, incomingTaskCount)) { + throw new ClusterManagerThrottlingException("Throttling Exception : Limit exceeded for " + taskThrottlingKey); + } + } + @Override public void onBeginSubmit(List tasks) { - ThrottlingKey clusterManagerThrottlingKey = ((ClusterStateTaskExecutor) tasks.get(0).batchingKey) + final ThrottlingKey clusterManagerThrottlingKey = ((ClusterStateTaskExecutor) tasks.get(0).batchingKey) .getClusterManagerThrottlingKey(); - tasksCount.putIfAbsent(clusterManagerThrottlingKey.getTaskThrottlingKey(), 0L); - tasksCount.computeIfPresent(clusterManagerThrottlingKey.getTaskThrottlingKey(), (key, count) -> { - int size = tasks.size(); - if (clusterManagerThrottlingKey.isThrottlingEnabled()) { - Long threshold = tasksThreshold.get(clusterManagerThrottlingKey.getTaskThrottlingKey()); - if (threshold != null && shouldThrottle(threshold, count, size)) { - clusterManagerTaskThrottlerListener.onThrottle(clusterManagerThrottlingKey.getTaskThrottlingKey(), size); - logger.warn( - "Throwing Throttling Exception for [{}]. Trying to add [{}] tasks to queue, limit is set to [{}]", - clusterManagerThrottlingKey.getTaskThrottlingKey(), - tasks.size(), - threshold - ); - throw new ClusterManagerThrottlingException( - "Throttling Exception : Limit exceeded for " + clusterManagerThrottlingKey.getTaskThrottlingKey() - ); - } - } - return count + size; - }); + final String taskThrottlingKey = clusterManagerThrottlingKey.getTaskThrottlingKey(); + final Long threshold = getThrottlingLimit(taskThrottlingKey); + final boolean isThrottlingEnabledWithThreshold = clusterManagerThrottlingKey.isThrottlingEnabled() && threshold != null; + int incomingTaskCount = tasks.size(); + + try { + tasksCount.putIfAbsent(taskThrottlingKey, 0L); + // Perform shallow check before acquiring lock to avoid blocking of network threads + // if throttling is ongoing for a specific task + failFastWhenThrottlingThresholdsAreAlreadyBreached( + isThrottlingEnabledWithThreshold, + threshold, + tasksCount.get(taskThrottlingKey), + incomingTaskCount, + taskThrottlingKey + ); + + tasksCount.computeIfPresent(taskThrottlingKey, (key, existingTaskCount) -> { + failFastWhenThrottlingThresholdsAreAlreadyBreached( + isThrottlingEnabledWithThreshold, + threshold, + existingTaskCount, + incomingTaskCount, + taskThrottlingKey + ); + return existingTaskCount + incomingTaskCount; + }); + } catch (final ClusterManagerThrottlingException e) { + clusterManagerTaskThrottlerListener.onThrottle(taskThrottlingKey, incomingTaskCount); + logger.trace( + "Throwing Throttling Exception for [{}]. Trying to add [{}] tasks to queue, limit is set to [{}]", + taskThrottlingKey, + incomingTaskCount, + threshold + ); + throw e; + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java index 04fa9fa45d5ea..7a835910c400f 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerThrottlingException.java @@ -25,4 +25,10 @@ public ClusterManagerThrottlingException(String msg, Object... args) { public ClusterManagerThrottlingException(StreamInput in) throws IOException { super(in); } + + @Override + public Throwable fillInStackTrace() { + // This is on the hot path; stack traces are expensive to compute and not very useful for this exception, so don't fill it. + return this; + } } diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterManagerTaskThrottlerTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterManagerTaskThrottlerTests.java index e25a0e0b2c3bf..3bd9333dc4168 100644 --- a/server/src/test/java/org/opensearch/cluster/service/ClusterManagerTaskThrottlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/ClusterManagerTaskThrottlerTests.java @@ -30,6 +30,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import static org.opensearch.test.ClusterServiceUtils.setState; @@ -69,7 +71,7 @@ public static void afterClass() { public void testDefaults() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); throttler.registerClusterManagerTask("create-index", true); @@ -108,7 +110,7 @@ public void testValidateSettingsForDifferentVersion() { } } - public void testValidateSettingsForTaskWihtoutRetryOnDataNode() { + public void testValidateSettingsForTaskWithoutRetryOnDataNode() { DiscoveryNode clusterManagerNode = getClusterManagerNode(Version.V_2_5_0); DiscoveryNode dataNode = getDataNode(Version.V_2_5_0); setState( @@ -139,7 +141,7 @@ public void testUpdateSettingsForNullValue() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); @@ -173,7 +175,7 @@ public void testSettingsOnBootstrap() { .put("cluster_manager.throttling.retry.max.delay", maxDelay + "s") .build(); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(initialSettings, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); @@ -187,7 +189,7 @@ public void testSettingsOnBootstrap() { public void testUpdateRetryDelaySetting() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); // verify defaults @@ -217,7 +219,7 @@ public void testValidateSettingsForUnknownTask() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); // set some limit for update snapshot tasks @@ -236,7 +238,7 @@ public void testUpdateThrottlingLimitForBasicSanity() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); @@ -263,7 +265,7 @@ public void testValidateSettingForLimit() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); @@ -274,7 +276,7 @@ public void testValidateSettingForLimit() { public void testUpdateLimit() { ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, new ClusterManagerThrottlingStats()); throttler.registerClusterManagerTask("put-mapping", true); @@ -309,7 +311,7 @@ public void testThrottlingForDisabledThrottlingTask() { String taskKey = "test"; ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, throttlingStats); ClusterManagerTaskThrottler.ThrottlingKey throttlingKey = throttler.registerClusterManagerTask(taskKey, false); @@ -321,6 +323,9 @@ public void testThrottlingForDisabledThrottlingTask() { // Asserting that there was not any throttling for it assertEquals(0L, throttlingStats.getThrottlingCount(taskKey)); + + // Asserting value in tasksCount map to make sure it gets updated even when throttling is disabled + assertEquals(Optional.of(10L).get(), throttler.tasksCount.get(taskKey)); } public void testThrottlingForInitialStaticSettingAndVersionCheck() { @@ -339,7 +344,7 @@ public void testThrottlingForInitialStaticSettingAndVersionCheck() { .put("cluster_manager.throttling.thresholds.put-mapping.value", put_mapping_threshold_value) .build(); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(initialSettings, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, throttlingStats); ClusterManagerTaskThrottler.ThrottlingKey throttlingKey = throttler.registerClusterManagerTask("put-mapping", true); @@ -367,7 +372,7 @@ public void testThrottling() { String taskKey = "test"; ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { - return clusterService.getMasterService().getMinNodeVersion(); + return clusterService.getClusterManagerService().getMinNodeVersion(); }, throttlingStats); ClusterManagerTaskThrottler.ThrottlingKey throttlingKey = throttler.registerClusterManagerTask(taskKey, true); @@ -406,6 +411,164 @@ public void testThrottling() { throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 1)); } + public void testThrottlingWithLock() { + ClusterManagerThrottlingStats throttlingStats = new ClusterManagerThrottlingStats(); + String taskKey = "test"; + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { + return clusterService.getClusterManagerService().getMinNodeVersion(); + }, throttlingStats); + ClusterManagerTaskThrottler.ThrottlingKey throttlingKey = throttler.registerClusterManagerTask(taskKey, true); + + throttler.updateLimit(taskKey, 5); + + // adding 3 tasks + throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)); + + // adding 3 more tasks, these tasks should be throttled + // taskCount in Queue: 3 Threshold: 5 + assertThrows( + ClusterManagerThrottlingException.class, + () -> throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)) + ); + assertEquals(3L, throttlingStats.getThrottlingCount(taskKey)); + + // remove one task + throttler.onBeginProcessing(getMockUpdateTaskList(taskKey, throttlingKey, 1)); + + // add 3 tasks should pass now. + // taskCount in Queue: 2 Threshold: 5 + throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)); + + final CountDownLatch latch = new CountDownLatch(1); + Thread threadToLock = null; + try { + // Taking lock on tasksCount will not impact throttling behaviour now. + threadToLock = new Thread(() -> { + throttler.tasksCount.computeIfPresent(taskKey, (key, count) -> { + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return 10L; + }); + }); + threadToLock.start(); + + // adding one task will throttle + // taskCount in Queue: 5 Threshold: 5 + final ClusterManagerThrottlingException exception = assertThrows( + ClusterManagerThrottlingException.class, + () -> throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 1)) + ); + assertEquals("Throttling Exception : Limit exceeded for test", exception.getMessage()); + assertEquals(Optional.of(5L).get(), throttler.tasksCount.get(taskKey)); + assertEquals(4L, throttlingStats.getThrottlingCount(taskKey)); + } finally { + if (threadToLock != null) { + latch.countDown(); + // Wait to complete and then assert on new tasksCount that got modified by threadToLock Thread + try { + threadToLock.join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + assertEquals(Optional.of(10L).get(), throttler.tasksCount.get(taskKey)); + } + + public void testThrottlingWithMultipleOnBeginSubmitsThreadsWithLock() { + ClusterManagerThrottlingStats throttlingStats = new ClusterManagerThrottlingStats(); + String taskKey = "test"; + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + ClusterManagerTaskThrottler throttler = new ClusterManagerTaskThrottler(Settings.EMPTY, clusterSettings, () -> { + return clusterService.getClusterManagerService().getMinNodeVersion(); + }, throttlingStats); + ClusterManagerTaskThrottler.ThrottlingKey throttlingKey = throttler.registerClusterManagerTask(taskKey, true); + + throttler.updateLimit(taskKey, 5); + + // adding 3 tasks + throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)); + + // adding 3 more tasks, these tasks should be throttled + // taskCount in Queue: 3 Threshold: 5 + assertThrows( + ClusterManagerThrottlingException.class, + () -> throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)) + ); + assertEquals(3L, throttlingStats.getThrottlingCount(taskKey)); + + // remove one task + throttler.onBeginProcessing(getMockUpdateTaskList(taskKey, throttlingKey, 1)); + + // add 3 tasks should pass now. + // taskCount in Queue: 2 Threshold: 5 + throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 3)); + + final CountDownLatch latch = new CountDownLatch(1); + Thread threadToLock = null; + List submittingThreads = new ArrayList<>(); + + try { + // Taking lock on tasksCount will not impact throttling behaviour now. + threadToLock = new Thread(() -> { + throttler.tasksCount.computeIfPresent(taskKey, (key, count) -> { + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return 10L; + }); + }); + threadToLock.start(); + + final CountDownLatch latch2 = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + Thread submittingThread = new Thread(() -> { + // adding one task will throttle + // taskCount in Queue: 5 Threshold: 5 + final ClusterManagerThrottlingException exception = assertThrows( + ClusterManagerThrottlingException.class, + () -> throttler.onBeginSubmit(getMockUpdateTaskList(taskKey, throttlingKey, 1)) + ); + assertEquals("Throttling Exception : Limit exceeded for test", exception.getMessage()); + assertEquals(Optional.of(5L).get(), throttler.tasksCount.get(taskKey)); + latch2.countDown(); + }); + submittingThread.start(); + submittingThreads.add(submittingThread); + } + try { + latch2.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + assertEquals(13L, throttlingStats.getThrottlingCount(taskKey)); + } finally { + if (threadToLock != null) { + latch.countDown(); + try { + // Wait to complete and then assert on new tasksCount that got modified by threadToLock Thread + threadToLock.join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + for (Thread submittingThread : submittingThreads) { + try { + submittingThread.join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + assertEquals(Optional.of(10L).get(), throttler.tasksCount.get(taskKey)); + } + private List getMockUpdateTaskList( String taskKey, ClusterManagerTaskThrottler.ThrottlingKey throttlingKey, From a608fcaefbf046fbc9f177907a14130d769cadda Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 4 Sep 2024 12:12:40 +0530 Subject: [PATCH 038/357] Changing checksum setting to support modes (#15622) * Changing checksum setting to support modes Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableServiceIT.java | 5 +- .../remote/RemoteStatePublicationIT.java | 5 +- .../common/settings/ClusterSettings.java | 2 +- .../remote/RemoteClusterStateService.java | 104 ++++-- .../RemoteClusterStateServiceTests.java | 333 +++++++++++++++++- 5 files changed, 406 insertions(+), 43 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java index 3c0c9a0611439..0a8c13adb034f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java @@ -67,7 +67,10 @@ protected Settings nodeSettings(int nodeOrdinal) { ) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) .put(REMOTE_PUBLICATION_EXPERIMENTAL, true) - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) + .put( + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), + RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE + ) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 3a3e023db6446..0e6321867a33b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -90,7 +90,10 @@ protected Settings nodeSettings(int nodeOrdinal) { .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) + .put( + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), + RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE + ) .build(); } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index c39040b503701..aef026f86c48d 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -740,7 +740,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, - RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 88dda3953a5f9..a223bfbe736c3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -71,6 +71,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -141,13 +142,49 @@ public class RemoteClusterStateService implements Closeable { Setting.Property.NodeScope ); - public static final Setting REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.state.checksum_validation.enabled", - false, - Property.Dynamic, - Property.NodeScope + public static final Setting REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING = new Setting<>( + "cluster.remote_store.state.checksum_validation.mode", + RemoteClusterStateValidationMode.NONE.name(), + RemoteClusterStateValidationMode::parseString, + Setting.Property.Dynamic, + Setting.Property.NodeScope ); + /** + * Validation mode for cluster state checksum. + * None: Validation will be disabled. + * Debug: Validation enabled but only matches checksum and logs failing entities. + * Trace: Matches checksum and downloads full cluster state to find diff in failing entities. Only logs failures. + * Failure: Throws exception on failing validation. + */ + public enum RemoteClusterStateValidationMode { + DEBUG("debug"), + TRACE("trace"), + FAILURE("failure"), + NONE("none"); + + public final String mode; + + RemoteClusterStateValidationMode(String mode) { + this.mode = mode; + } + + public static RemoteClusterStateValidationMode parseString(String mode) { + try { + return RemoteClusterStateValidationMode.valueOf(mode.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "[" + + mode + + "] mode is not supported. " + + "supported modes are [" + + Arrays.toString(RemoteClusterStateValidationMode.values()) + + "]" + ); + } + } + } + private TimeValue remoteStateReadTimeout; private final String nodeId; private final Supplier repositoriesService; @@ -159,7 +196,7 @@ public class RemoteClusterStateService implements Closeable { private BlobStoreTransferService blobStoreTransferService; private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; - private boolean checksumValidationEnabled; + private RemoteClusterStateValidationMode remoteClusterStateValidationMode; private final RemotePersistenceStats remoteStateStats; private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; @@ -206,11 +243,8 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteStateReadTimeout); - this.checksumValidationEnabled = clusterSettings.get(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING); - clusterSettings.addSettingsUpdateConsumer( - REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING, - this::setChecksumValidationEnabled - ); + this.remoteClusterStateValidationMode = REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING, this::setChecksumValidationMode); this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; @@ -272,7 +306,7 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat uploadedMetadataResults, previousClusterUUID, clusterStateDiffManifest, - checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, false, codecVersion ); @@ -472,7 +506,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), clusterStateDiffManifest, - checksumValidationEnabled ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, false, previousManifest.getCodecVersion() ); @@ -917,7 +951,7 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted(ClusterState clus uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), - checksumValidationEnabled ? previousManifest.getClusterStateChecksum() : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, true, previousManifest.getCodecVersion() ); @@ -1003,8 +1037,8 @@ private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } - private void setChecksumValidationEnabled(Boolean checksumValidationEnabled) { - this.checksumValidationEnabled = checksumValidationEnabled; + private void setChecksumValidationMode(RemoteClusterStateValidationMode remoteClusterStateValidationMode) { + this.remoteClusterStateValidationMode = remoteClusterStateValidationMode; } // Package private for unit test @@ -1376,7 +1410,9 @@ public ClusterState getClusterStateForManifest( includeEphemeral ); - if (includeEphemeral && checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { + if (includeEphemeral + && !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); } } else { @@ -1498,7 +1534,7 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); - if (checksumValidationEnabled && manifest.getClusterStateChecksum() != null) { + if (!remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); } final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -1517,20 +1553,24 @@ void validateClusterStateFromChecksum( ) { ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); - if (!failedValidation.isEmpty()) { - logger.error( - () -> new ParameterizedMessage( - "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", - manifest.getClusterStateChecksum(), - newClusterStateChecksum, - failedValidation - ) + if (failedValidation.isEmpty()) { + return; + } + logger.error( + () -> new ParameterizedMessage( + "Cluster state checksums do not match. Checksum from manifest {}, checksum from created cluster state {}. Entities failing validation {}", + manifest.getClusterStateChecksum(), + newClusterStateChecksum, + failedValidation + ) + ); + if (isFullStateDownload && remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) { + throw new IllegalStateException( + "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation ); - if (isFullStateDownload) { - throw new IllegalStateException( - "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation - ); - } + } + if (remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE) + || remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.TRACE)) { // download full cluster state and match against state created for the failing entities ClusterState fullClusterState = readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), @@ -1663,6 +1703,8 @@ void validateClusterStateFromChecksum( break; } } + } + if (remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) { throw new IllegalStateException( "Cluster state checksums do not match during diff read. Validation failed for " + failedValidation ); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index ccadbfb3fbebc..15bdaf03b5990 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -2959,12 +2959,12 @@ private void initializeRoutingTable() { ); } - private void initializeWithChecksumEnabled() { + private void initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode mode) { Settings newSettings = Settings.builder() .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true) + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), mode.name()) .build(); clusterSettings.applySettings(newSettings); @@ -2991,7 +2991,7 @@ private void initializeWithChecksumEnabled() { } public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); mockBlobStoreObjects(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -3034,8 +3034,51 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); } + public void testWriteFullMetadataSuccessWithChecksumValidationModeNone() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); + mockBlobStoreObjects(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( + clusterState, + "prev-cluster-uuid", + MANIFEST_CURRENT_CODEC_VERSION + ).getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1L) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertNull(manifest.getClusterStateChecksum()); + } + public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); @@ -3086,8 +3129,60 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t assertThat(manifest.getClusterStateChecksum(), is(expectedManifest.getClusterStateChecksum())); } + public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + mockBlobStoreObjects(); + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); + final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) + .build(); + + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .indices(Collections.emptyList()) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); + + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata( + previousClusterState, + clusterState, + previousManifest + ).getClusterMetadataManifest(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( + "test-index", + "index-uuid", + "routing-filename", + INDEX_ROUTING_METADATA_PREFIX + ); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of(uploadedIndexMetadata)) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) + .checksum(new ClusterStateChecksum(clusterState)) + .build(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertThat(manifest.getRoutingTableVersion(), is(expectedManifest.getRoutingTableVersion())); + assertThat(manifest.getIndicesRouting().get(0).getIndexName(), is(uploadedIndiceRoutingMetadata.getIndexName())); + assertThat(manifest.getIndicesRouting().get(0).getIndexUUID(), is(uploadedIndiceRoutingMetadata.getIndexUUID())); + assertThat(manifest.getIndicesRouting().get(0).getUploadedFilename(), notNullValue()); + assertNull(manifest.getClusterStateChecksum()); + } + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullChecksum() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().build(); mockBlobStoreObjects(); remoteClusterStateService.start(); @@ -3145,7 +3240,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) @@ -3176,8 +3271,40 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); } + public void testGetClusterStateForManifestWithChecksumValidationModeNone() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(0)).validateClusterStateFromChecksum(any(), any(), anyString(), anyString(), anyBoolean()); + } + public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) @@ -3218,8 +3345,95 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma ); } + public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException { + initializeWithChecksumEnabled( + randomFrom( + Arrays.asList( + RemoteClusterStateService.RemoteClusterStateValidationMode.DEBUG, + RemoteClusterStateService.RemoteClusterStateValidationMode.TRACE + ) + ) + ); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).build(); + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); + verify(mockService, times(1)).validateClusterStateFromChecksum( + manifest, + clusterStateWrong, + ClusterName.DEFAULT.value(), + NODE_ID, + true + ); + } + public void testGetClusterStateUsingDiffWithChecksum() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); + + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + + public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) @@ -3250,6 +3464,107 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { ); mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); + verify(mockService, times(0)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + + public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.DEBUG); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); + verify(mockService, times(1)).validateClusterStateFromChecksum( + eq(manifest), + any(ClusterState.class), + eq(ClusterName.DEFAULT.value()), + eq(NODE_ID), + eq(false) + ); + } + + public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws IOException { + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.TRACE); + ClusterState clusterState = generateClusterStateWithAllAttributes().build(); + ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( + new ClusterStateChecksum(clusterState) + ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); + + remoteClusterStateService.start(); + RemoteClusterStateService mockService = spy(remoteClusterStateService); + ClusterState clusterStateWrong = ClusterState.builder(clusterState).routingTable(RoutingTable.EMPTY_ROUTING_TABLE).build(); + doReturn(clusterStateWrong).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(emptyList()), + eq(emptyMap()), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + anyBoolean(), + eq(emptyList()), + anyBoolean(), + eq(emptyMap()), + anyBoolean(), + anyBoolean() + ); + doReturn(clusterState).when(mockService) + .readClusterStateInParallel( + any(), + eq(manifest), + eq(manifest.getClusterUUID()), + eq(NODE_ID), + eq(manifest.getIndices()), + eq(manifest.getCustomMetadataMap()), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(true), + eq(manifest.getIndicesRouting()), + eq(true), + eq(manifest.getClusterStateCustomMap()), + eq(false), + eq(true) + ); + + mockService.getClusterStateUsingDiff(manifest, clusterState, NODE_ID); verify(mockService, times(1)).validateClusterStateFromChecksum( eq(manifest), any(ClusterState.class), @@ -3260,7 +3575,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { } public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { - initializeWithChecksumEnabled(); + initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( new ClusterStateChecksum(clusterState) From f7b78d1f5ac8e0a9e8e37ef060badd8f3b22dc05 Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Wed, 4 Sep 2024 12:16:21 +0530 Subject: [PATCH 039/357] Fix version check for BWC of Discovery Node serialization (#15620) Signed-off-by: RS146BIJAY --- .../main/java/org/opensearch/cluster/node/DiscoveryNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 8ac978a72b21b..3ed7633dce189 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -360,7 +360,7 @@ public DiscoveryNode(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { writeToUtil(out, false); } else { writeToUtil(out, true); From e08727289cbbebda27fad118c4624accdbcfae68 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 4 Sep 2024 13:27:46 +0530 Subject: [PATCH 040/357] Initialise RemoteRoutingTableService when we have RoutingTable repository configured (#15645) Signed-off-by: Shivansh Arora --- .../node/remotestore/RemoteStoreNodeAttribute.java | 5 +---- .../remote/RemoteClusterStateServiceTests.java | 11 ++++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a0f745a4270c4..efaefd530b34d 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -13,7 +13,6 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -29,8 +28,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; - /** * This is an abstraction for validating and storing information specific to remote backed storage nodes. * @@ -202,7 +199,7 @@ private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { } public static boolean isRemoteRoutingTableEnabled(Settings settings) { - return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && isRemoteRoutingTableAttributePresent(settings); + return isRemoteRoutingTableAttributePresent(settings); } public RepositoriesMetadata getRepositoriesMetadata() { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 15bdaf03b5990..21b88e5bd66b9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -151,6 +151,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -485,8 +486,8 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); - assertEquals(7, actionListenerArgumentCaptor.getAllValues().size()); - assertEquals(7, writeContextArgumentCaptor.getAllValues().size()); + assertEquals(8, actionListenerArgumentCaptor.getAllValues().size()); + assertEquals(8, writeContextArgumentCaptor.getAllValues().size()); byte[] writtenBytes = capturedWriteContext.get("metadata") .getStreamProvider(Integer.MAX_VALUE) @@ -722,7 +723,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { eq(false), eq(Collections.emptyMap()), eq(false), - eq(Collections.emptyList()), + anyList(), Mockito.any(StringKeyDiffProvider.class) ); @@ -743,7 +744,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { assertThat(manifest.getTemplatesMetadata(), notNullValue()); assertThat(manifest.getCoordinationMetadata(), notNullValue()); assertThat(manifest.getCustomMetadataMap().size(), is(2)); - assertThat(manifest.getIndicesRouting().size(), is(0)); + assertThat(manifest.getIndicesRouting().size(), is(1)); } public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws IOException { @@ -2642,7 +2643,7 @@ public void testRemoteStateUploadStats() throws IOException { } public void testRemoteRoutingTableNotInitializedWhenDisabled() { - if (publicationEnabled) { + if (isRemoteRoutingTableEnabled(settings)) { assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof InternalRemoteRoutingTableService); } else { assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof NoopRemoteRoutingTableService); From b7c7f64c156311ea1e99baaf3011de6c335679fb Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:04:13 +0530 Subject: [PATCH 041/357] Reset discovery nodes in most transport node actions request. (#15131) Signed-off-by: Swetha Guptha Co-authored-by: Swetha Guptha --- CHANGELOG.md | 1 + .../hotthreads/NodesHotThreadsRequest.java | 2 +- .../cluster/node/info/NodesInfoRequest.java | 2 +- .../cluster/node/stats/NodesStatsRequest.java | 4 +- .../cluster/node/usage/NodesUsageRequest.java | 2 +- .../status/TransportNodesSnapshotsStatus.java | 2 +- .../cluster/stats/ClusterStatsRequest.java | 2 +- .../find/FindDanglingIndexRequest.java | 2 +- .../list/ListDanglingIndicesRequest.java | 4 +- .../action/search/GetAllPitNodesRequest.java | 2 +- .../support/nodes/BaseNodesRequest.java | 16 ++++-- .../support/nodes/TransportNodesAction.java | 12 ++--- .../TransportNodesListGatewayMetaState.java | 2 +- ...ransportNodesListGatewayStartedShards.java | 2 +- ...ortNodesListGatewayStartedShardsBatch.java | 2 +- .../TransportNodesListShardStoreMetadata.java | 2 +- ...sportNodesListShardStoreMetadataBatch.java | 2 +- .../admin/cluster/RestClusterStatsAction.java | 1 - .../admin/cluster/RestNodesInfoAction.java | 1 - .../admin/cluster/RestNodesStatsAction.java | 1 - .../rest/action/cat/RestNodesAction.java | 2 - .../TransportClusterStatsActionTests.java | 37 ------------- .../nodes/TransportNodesActionTests.java | 52 ++++++++++++++++++- .../nodes/TransportNodesInfoActionTests.java | 20 ------- .../nodes/TransportNodesStatsActionTests.java | 24 +-------- 25 files changed, 87 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa25dc88a64d7..0e255ad694206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) +- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java index 9e52b90f7bd38..b7b60732fcbfc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java @@ -70,7 +70,7 @@ public NodesHotThreadsRequest(StreamInput in) throws IOException { * threads for all nodes is used. */ public NodesHotThreadsRequest(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); } public int threads() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java index 17b633c533218..26b53e8db642f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -72,7 +72,7 @@ public NodesInfoRequest(StreamInput in) throws IOException { * for all nodes will be returned. */ public NodesInfoRequest(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); defaultMetrics(); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 379836cf442e3..f1f9f93afdad2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -58,7 +58,7 @@ public class NodesStatsRequest extends BaseNodesRequest { private final Set requestedMetrics = new HashSet<>(); public NodesStatsRequest() { - super((String[]) null); + super(false, (String[]) null); } public NodesStatsRequest(StreamInput in) throws IOException { @@ -74,7 +74,7 @@ public NodesStatsRequest(StreamInput in) throws IOException { * for all nodes will be returned. */ public NodesStatsRequest(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); } /** diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java index 385f48d5690c1..955f77921009f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/usage/NodesUsageRequest.java @@ -61,7 +61,7 @@ public NodesUsageRequest(StreamInput in) throws IOException { * passed, usage for all nodes will be returned. */ public NodesUsageRequest(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); } /** diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index 8f76dbce048cd..3654cab3c5a47 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -161,7 +161,7 @@ public Request(StreamInput in) throws IOException { } public Request(String[] nodesIds) { - super(nodesIds); + super(false, nodesIds); } public Request snapshots(Snapshot[] snapshots) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index bd75b2210e474..b82a9d256a134 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -62,7 +62,7 @@ public ClusterStatsRequest(StreamInput in) throws IOException { * based on all nodes will be returned. */ public ClusterStatsRequest(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); } public boolean useAggregatedNodeLevelResponses() { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java index f853a47b3c2bf..b38cb46b23ace 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java @@ -53,7 +53,7 @@ public FindDanglingIndexRequest(StreamInput in) throws IOException { } public FindDanglingIndexRequest(String indexUUID) { - super(Strings.EMPTY_ARRAY); + super(false, Strings.EMPTY_ARRAY); this.indexUUID = indexUUID; } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java index 119c4acbf4160..0076c9e294b69 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java @@ -58,12 +58,12 @@ public ListDanglingIndicesRequest(StreamInput in) throws IOException { } public ListDanglingIndicesRequest() { - super(Strings.EMPTY_ARRAY); + super(false, Strings.EMPTY_ARRAY); this.indexUUID = null; } public ListDanglingIndicesRequest(String indexUUID) { - super(Strings.EMPTY_ARRAY); + super(false, Strings.EMPTY_ARRAY); this.indexUUID = indexUUID; } diff --git a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java index 336c8139561e9..97c0d66f4b8e1 100644 --- a/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java +++ b/server/src/main/java/org/opensearch/action/search/GetAllPitNodesRequest.java @@ -27,7 +27,7 @@ public class GetAllPitNodesRequest extends BaseNodesRequest * Setting default behavior as `true` but can be explicitly changed in requests that do not require. */ private boolean includeDiscoveryNodes = true; + private final TimeValue DEFAULT_TIMEOUT_SECS = TimeValue.timeValueSeconds(30); private TimeValue timeout; @@ -88,11 +89,22 @@ protected BaseNodesRequest(String... nodesIds) { this.nodesIds = nodesIds; } + protected BaseNodesRequest(boolean includeDiscoveryNodes, String... nodesIds) { + this.nodesIds = nodesIds; + this.includeDiscoveryNodes = includeDiscoveryNodes; + } + protected BaseNodesRequest(DiscoveryNode... concreteNodes) { this.nodesIds = null; this.concreteNodes = concreteNodes; } + protected BaseNodesRequest(boolean includeDiscoveryNodes, DiscoveryNode... concreteNodes) { + this.nodesIds = null; + this.concreteNodes = concreteNodes; + this.includeDiscoveryNodes = includeDiscoveryNodes; + } + public final String[] nodesIds() { return nodesIds; } @@ -127,10 +139,6 @@ public void setConcreteNodes(DiscoveryNode[] concreteNodes) { this.concreteNodes = concreteNodes; } - public void setIncludeDiscoveryNodes(boolean value) { - includeDiscoveryNodes = value; - } - public boolean getIncludeDiscoveryNodes() { return includeDiscoveryNodes; } diff --git a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java index 3acd12f632e0f..dccd5059dd52d 100644 --- a/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/opensearch/action/support/nodes/TransportNodesAction.java @@ -240,18 +240,16 @@ class AsyncAction { } this.responses = new AtomicReferenceArray<>(request.concreteNodes().length); this.concreteNodes = request.concreteNodes(); - if (request.getIncludeDiscoveryNodes() == false) { - // As we transfer the ownership of discovery nodes to route the request to into the AsyncAction class, we - // remove the list of DiscoveryNodes from the request. This reduces the payload of the request and improves + // As we transfer the ownership of discovery nodes to route the request to into the AsyncAction class, + // we remove the list of DiscoveryNodes from the request. This reduces the payload of the request and improves // the number of concrete nodes in the memory. request.setConcreteNodes(null); } } void start() { - final DiscoveryNode[] nodes = this.concreteNodes; - if (nodes.length == 0) { + if (this.concreteNodes.length == 0) { // nothing to notify threadPool.generic().execute(() -> listener.onResponse(newResponse(request, responses))); return; @@ -260,9 +258,9 @@ void start() { if (request.timeout() != null) { builder.withTimeout(request.timeout()); } - for (int i = 0; i < nodes.length; i++) { + for (int i = 0; i < this.concreteNodes.length; i++) { final int idx = i; - final DiscoveryNode node = nodes[i]; + final DiscoveryNode node = this.concreteNodes[i]; final String nodeId = node.getId(); try { TransportRequest nodeRequest = newNodeRequest(request); diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java index 647e3632ea0ca..7f634534d1ed6 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayMetaState.java @@ -133,7 +133,7 @@ public Request(StreamInput in) throws IOException { } public Request(String... nodesIds) { - super(nodesIds); + super(false, nodesIds); } } diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java index 4b1f611bb88ab..b9cfada1910ce 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java @@ -197,7 +197,7 @@ public Request(StreamInput in) throws IOException { } public Request(ShardId shardId, String customDataPath, DiscoveryNode[] nodes) { - super(nodes); + super(false, nodes); this.shardId = Objects.requireNonNull(shardId); this.customDataPath = Objects.requireNonNull(customDataPath); } diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java index 89362988b4d85..45ead58f9f195 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java @@ -182,7 +182,7 @@ public Request(StreamInput in) throws IOException { } public Request(DiscoveryNode[] nodes, Map shardAttributes) { - super(nodes); + super(false, nodes); this.shardAttributes = Objects.requireNonNull(shardAttributes); } diff --git a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java index eeee5d8a409aa..2f666f67280bd 100644 --- a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java +++ b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadata.java @@ -176,7 +176,7 @@ public Request(StreamInput in) throws IOException { } public Request(ShardId shardId, String customDataPath, DiscoveryNode[] nodes) { - super(nodes); + super(false, nodes); this.shardId = Objects.requireNonNull(shardId); this.customDataPath = Objects.requireNonNull(customDataPath); } diff --git a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java index 22b03539cca74..2c84ec31b987d 100644 --- a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java +++ b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java @@ -188,7 +188,7 @@ public Request(StreamInput in) throws IOException { } public Request(Map shardAttributes, DiscoveryNode[] nodes) { - super(nodes); + super(false, nodes); this.shardAttributes = Objects.requireNonNull(shardAttributes); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index d4426a004af8e..ee33bd18db05d 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -66,7 +66,6 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); clusterStatsRequest.timeout(request.param("timeout")); - clusterStatsRequest.setIncludeDiscoveryNodes(false); clusterStatsRequest.useAggregatedNodeLevelResponses(true); return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesInfoAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesInfoAction.java index 4ac51933ea382..37e4c8783d0df 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesInfoAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesInfoAction.java @@ -88,7 +88,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final NodesInfoRequest nodesInfoRequest = prepareRequest(request); nodesInfoRequest.timeout(request.param("timeout")); settingsFilter.addFilterSettingParams(request); - nodesInfoRequest.setIncludeDiscoveryNodes(false); return channel -> client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel)); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java index 0119731e4a0d7..0a0ee84b1999f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java @@ -232,7 +232,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC // If no levels are passed in this results in an empty array. String[] levels = Strings.splitStringByCommaToArray(request.param("level")); nodesStatsRequest.indices().setLevels(levels); - nodesStatsRequest.setIncludeDiscoveryNodes(false); nodesStatsRequest.indices().setIncludeIndicesStatsByLevel(true); return channel -> client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel)); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 1aa40b50290cd..26d6b91882cbd 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -125,7 +125,6 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli public void processResponse(final ClusterStateResponse clusterStateResponse) { NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(); nodesInfoRequest.timeout(request.param("timeout")); - nodesInfoRequest.setIncludeDiscoveryNodes(false); nodesInfoRequest.clear() .addMetrics( NodesInfoRequest.Metric.JVM.metricName(), @@ -138,7 +137,6 @@ public void processResponse(final ClusterStateResponse clusterStateResponse) { public void processResponse(final NodesInfoResponse nodesInfoResponse) { NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(); nodesStatsRequest.timeout(request.param("timeout")); - nodesStatsRequest.setIncludeDiscoveryNodes(false); nodesStatsRequest.clear() .indices(true) .addMetrics( diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportClusterStatsActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportClusterStatsActionTests.java index f8e14b477b8ef..fc920b847cef9 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportClusterStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportClusterStatsActionTests.java @@ -33,48 +33,12 @@ public class TransportClusterStatsActionTests extends TransportNodesActionTests { - /** - * By default, we send discovery nodes list to each request that is sent across from the coordinator node. This - * behavior is asserted in this test. - */ - public void testClusterStatsActionWithRetentionOfDiscoveryNodesList() { - ClusterStatsRequest request = new ClusterStatsRequest(); - request.setIncludeDiscoveryNodes(true); - Map> combinedSentRequest = performNodesInfoAction(request); - - assertNotNull(combinedSentRequest); - combinedSentRequest.forEach((node, capturedRequestList) -> { - assertNotNull(capturedRequestList); - capturedRequestList.forEach(sentRequest -> { - assertNotNull(sentRequest.getDiscoveryNodes()); - assertEquals(sentRequest.getDiscoveryNodes().length, clusterService.state().nodes().getSize()); - }); - }); - } - - public void testClusterStatsActionWithPreFilledConcreteNodesAndWithRetentionOfDiscoveryNodesList() { - ClusterStatsRequest request = new ClusterStatsRequest(); - Collection discoveryNodes = clusterService.state().getNodes().getNodes().values(); - request.setConcreteNodes(discoveryNodes.toArray(DiscoveryNode[]::new)); - Map> combinedSentRequest = performNodesInfoAction(request); - - assertNotNull(combinedSentRequest); - combinedSentRequest.forEach((node, capturedRequestList) -> { - assertNotNull(capturedRequestList); - capturedRequestList.forEach(sentRequest -> { - assertNotNull(sentRequest.getDiscoveryNodes()); - assertEquals(sentRequest.getDiscoveryNodes().length, clusterService.state().nodes().getSize()); - }); - }); - } - /** * In the optimized ClusterStats Request, we do not send the DiscoveryNodes List to each node. This behavior is * asserted in this test. */ public void testClusterStatsActionWithoutRetentionOfDiscoveryNodesList() { ClusterStatsRequest request = new ClusterStatsRequest(); - request.setIncludeDiscoveryNodes(false); Map> combinedSentRequest = performNodesInfoAction(request); assertNotNull(combinedSentRequest); @@ -88,7 +52,6 @@ public void testClusterStatsActionWithPreFilledConcreteNodesAndWithoutRetentionO ClusterStatsRequest request = new ClusterStatsRequest(); Collection discoveryNodes = clusterService.state().getNodes().getNodes().values(); request.setConcreteNodes(discoveryNodes.toArray(DiscoveryNode[]::new)); - request.setIncludeDiscoveryNodes(false); Map> combinedSentRequest = performNodesInfoAction(request); assertNotNull(combinedSentRequest); diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java index 7e968aa8fb199..a338e68276bbc 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesActionTests.java @@ -62,6 +62,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -71,6 +72,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.opensearch.test.ClusterServiceUtils.setState; @@ -166,6 +168,36 @@ public void testCustomResolving() throws Exception { assertEquals(clusterService.state().nodes().getDataNodes().size(), capturedRequests.size()); } + public void testTransportNodesActionWithDiscoveryNodesIncluded() { + String[] nodeIds = clusterService.state().nodes().getNodes().keySet().toArray(new String[0]); + TestNodesRequest request = new TestNodesRequest(true, nodeIds); + getTestTransportNodesAction().new AsyncAction(null, request, new PlainActionFuture<>()).start(); + Map> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear(); + List capturedTransportNodeRequestList = capturedRequests.values() + .stream() + .flatMap(Collection::stream) + .map(capturedRequest -> (TestNodeRequest) capturedRequest.request) + .collect(Collectors.toList()); + assertEquals(nodeIds.length, capturedTransportNodeRequestList.size()); + capturedTransportNodeRequestList.forEach( + capturedRequest -> assertEquals(nodeIds.length, capturedRequest.testNodesRequest.concreteNodes().length) + ); + } + + public void testTransportNodesActionWithDiscoveryNodesReset() { + String[] nodeIds = clusterService.state().nodes().getNodes().keySet().toArray(new String[0]); + TestNodesRequest request = new TestNodesRequest(false, nodeIds); + getTestTransportNodesAction().new AsyncAction(null, request, new PlainActionFuture<>()).start(); + Map> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear(); + List capturedTransportNodeRequestList = capturedRequests.values() + .stream() + .flatMap(Collection::stream) + .map(capturedRequest -> (TestNodeRequest) capturedRequest.request) + .collect(Collectors.toList()); + assertEquals(nodeIds.length, capturedTransportNodeRequestList.size()); + capturedTransportNodeRequestList.forEach(capturedRequest -> assertNull(capturedRequest.testNodesRequest.concreteNodes())); + } + private List mockList(Supplier supplier, int size) { List failures = new ArrayList<>(size); for (int i = 0; i < size; ++i) { @@ -314,7 +346,7 @@ protected TestNodesResponse newResponse( @Override protected TestNodeRequest newNodeRequest(TestNodesRequest request) { - return new TestNodeRequest(); + return new TestNodeRequest(request); } @Override @@ -357,6 +389,10 @@ private static class TestNodesRequest extends BaseNodesRequest TestNodesRequest(String... nodesIds) { super(nodesIds); } + + TestNodesRequest(boolean includeDiscoveryNodes, String... nodesIds) { + super(includeDiscoveryNodes, nodesIds); + } } private static class TestNodesResponse extends BaseNodesResponse { @@ -385,10 +421,24 @@ protected void writeNodesTo(StreamOutput out, List nodes) thro } private static class TestNodeRequest extends TransportRequest { + + protected TestNodesRequest testNodesRequest; + TestNodeRequest() {} + TestNodeRequest(TestNodesRequest testNodesRequest) { + this.testNodesRequest = testNodesRequest; + } + TestNodeRequest(StreamInput in) throws IOException { super(in); + testNodesRequest = new TestNodesRequest(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + testNodesRequest.writeTo(out); } } diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesInfoActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesInfoActionTests.java index e9e09d0dbbbf9..8277dcd363a8d 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesInfoActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesInfoActionTests.java @@ -31,32 +31,12 @@ public class TransportNodesInfoActionTests extends TransportNodesActionTests { - /** - * By default, we send discovery nodes list to each request that is sent across from the coordinator node. This - * behavior is asserted in this test. - */ - public void testNodesInfoActionWithRetentionOfDiscoveryNodesList() { - NodesInfoRequest request = new NodesInfoRequest(); - request.setIncludeDiscoveryNodes(true); - Map> combinedSentRequest = performNodesInfoAction(request); - - assertNotNull(combinedSentRequest); - combinedSentRequest.forEach((node, capturedRequestList) -> { - assertNotNull(capturedRequestList); - capturedRequestList.forEach(sentRequest -> { - assertNotNull(sentRequest.getDiscoveryNodes()); - assertEquals(sentRequest.getDiscoveryNodes().length, clusterService.state().nodes().getSize()); - }); - }); - } - /** * In the optimized ClusterStats Request, we do not send the DiscoveryNodes List to each node. This behavior is * asserted in this test. */ public void testNodesInfoActionWithoutRetentionOfDiscoveryNodesList() { NodesInfoRequest request = new NodesInfoRequest(); - request.setIncludeDiscoveryNodes(false); Map> combinedSentRequest = performNodesInfoAction(request); assertNotNull(combinedSentRequest); diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesStatsActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesStatsActionTests.java index c7c420e353e1a..5e74dcdbc4953 100644 --- a/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportNodesStatsActionTests.java @@ -31,31 +31,11 @@ public class TransportNodesStatsActionTests extends TransportNodesActionTests { /** - * By default, we send discovery nodes list to each request that is sent across from the coordinator node. This - * behavior is asserted in this test. - */ - public void testNodesStatsActionWithRetentionOfDiscoveryNodesList() { - NodesStatsRequest request = new NodesStatsRequest(); - request.setIncludeDiscoveryNodes(true); - Map> combinedSentRequest = performNodesStatsAction(request); - - assertNotNull(combinedSentRequest); - combinedSentRequest.forEach((node, capturedRequestList) -> { - assertNotNull(capturedRequestList); - capturedRequestList.forEach(sentRequest -> { - assertNotNull(sentRequest.getDiscoveryNodes()); - assertEquals(sentRequest.getDiscoveryNodes().length, clusterService.state().nodes().getSize()); - }); - }); - } - - /** - * By default, we send discovery nodes list to each request that is sent across from the coordinator node. This - * behavior is asserted in this test. + * We don't want to send discovery nodes list to each request that is sent across from the coordinator node. + * This behavior is asserted in this test. */ public void testNodesStatsActionWithoutRetentionOfDiscoveryNodesList() { NodesStatsRequest request = new NodesStatsRequest(); - request.setIncludeDiscoveryNodes(false); Map> combinedSentRequest = performNodesStatsAction(request); assertNotNull(combinedSentRequest); From 48cf5f061094d89aaa564b97667f4965556a8e49 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:00:38 +0530 Subject: [PATCH 042/357] Mute failing snapshot status tests (#15652) Signed-off-by: Lakshya Taragi --- .../test/snapshot.status/10_basic.yml | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index c35f2419bdc91..819f04407b219 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -25,38 +25,40 @@ setup: snapshot: test_snapshot wait_for_completion: true - - do: - snapshot.status: - repository: test_repo_status_1 - snapshot: test_snapshot - - - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot } - - match: { snapshots.0.state: SUCCESS } - - gt: { snapshots.0.stats.incremental.file_count: 0 } - - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } - - gt: { snapshots.0.stats.total.file_count: 0 } - - gt: { snapshots.0.stats.total.size_in_bytes: 0 } - - is_true: snapshots.0.stats.start_time_in_millis -## fast in memory snapshots can take less than one millisecond to complete. - - gte: { snapshots.0.stats.time_in_millis: 0 } - ---- -"Get missing snapshot status throws an exception": - - - do: - catch: /snapshot_missing_exception.+ is missing/ - snapshot.status: - repository: test_repo_status_1 - snapshot: test_nonexistent_snapshot - ---- -"Get missing snapshot status succeeds when ignoreUnavailable is true": - - - do: - snapshot.status: - repository: test_repo_status_1 - snapshot: test_nonexistent_snapshot - ignore_unavailable: true +# TODO: fix and unmute tests - - is_true: snapshots +# - do: +# snapshot.status: +# repository: test_repo_status_1 +# snapshot: test_snapshot +# +# - is_true: snapshots +# - match: { snapshots.0.snapshot: test_snapshot } +# - match: { snapshots.0.state: SUCCESS } +# - gt: { snapshots.0.stats.incremental.file_count: 0 } +# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } +# - gt: { snapshots.0.stats.total.file_count: 0 } +# - gt: { snapshots.0.stats.total.size_in_bytes: 0 } +# - is_true: snapshots.0.stats.start_time_in_millis +### fast in memory snapshots can take less than one millisecond to complete. +# - gte: { snapshots.0.stats.time_in_millis: 0 } +# +#--- +#"Get missing snapshot status throws an exception": +# +# - do: +# catch: /snapshot_missing_exception.+ is missing/ +# snapshot.status: +# repository: test_repo_status_1 +# snapshot: test_nonexistent_snapshot +# +#--- +#"Get missing snapshot status succeeds when ignoreUnavailable is true": +# +# - do: +# snapshot.status: +# repository: test_repo_status_1 +# snapshot: test_nonexistent_snapshot +# ignore_unavailable: true +# +# - is_true: snapshots From 23f5c2f988a7e78c3438ad2aeacd029f7a9e3ba6 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 4 Sep 2024 15:03:56 +0530 Subject: [PATCH 043/357] [Star Tree] Scaled Float Support (#15442) --------- Signed-off-by: Sarthak Aggarwal --- .../index/mapper/ScaledFloatFieldMapper.java | 23 +++- .../mapper/ScaledFloatFieldMapperTests.java | 108 +++++++++++++++-- .../index/mapper/StarTreeMapperIT.java | 73 +++++------- .../datacube/DimensionFactory.java | 12 +- .../datacube/DimensionType.java | 31 +++++ .../aggregators/CountValueAggregator.java | 9 +- .../aggregators/DocCountAggregator.java | 7 +- .../aggregators/MaxValueAggregator.java | 6 +- .../aggregators/MetricAggregatorInfo.java | 15 ++- .../aggregators/MinValueAggregator.java | 6 +- .../StatelessDoubleValueAggregator.java | 17 +-- .../aggregators/SumValueAggregator.java | 19 +-- .../startree/aggregators/ValueAggregator.java | 4 +- .../aggregators/ValueAggregatorFactory.java | 12 +- .../numerictype/StarTreeNumericType.java | 66 ----------- .../StarTreeNumericTypeConverters.java | 58 --------- .../aggregators/numerictype/package-info.java | 14 --- .../builder/AbstractDocumentsFileManager.java | 67 ++++++----- .../startree/builder/BaseStarTreeBuilder.java | 41 ++++--- .../node/FixedLengthStarTreeNode.java | 2 +- .../index/mapper/DateFieldMapper.java | 1 + .../index/mapper/DocCountFieldMapper.java | 5 + .../index/mapper/FieldValueConverter.java | 30 +++++ .../org/opensearch/index/mapper/Mapper.java | 2 +- .../index/mapper/MapperBuilderProperties.java | 41 +++++++ .../index/mapper/NumberFieldMapper.java | 77 +++++++++++- .../index/mapper/StarTreeMapper.java | 9 +- .../StarTreeDocValuesFormatTests.java | 4 +- .../datacube/startree/StarTreeTestUtils.java | 14 +-- .../AbstractValueAggregatorTests.java | 23 ++-- .../CountValueAggregatorTests.java | 8 +- .../aggregators/DocCountAggregatorTests.java | 10 +- .../aggregators/MaxValueAggregatorTests.java | 19 +-- .../MetricAggregatorInfoTests.java | 29 +++-- .../aggregators/MinValueAggregatorTests.java | 19 +-- .../StaticValueAggregatorTests.java | 12 +- .../aggregators/SumValueAggregatorTests.java | 16 +-- .../ValueAggregatorFactoryTests.java | 12 +- .../builder/AbstractStarTreeBuilderTests.java | 8 +- .../builder/BaseStarTreeBuilderTests.java | 5 +- .../meta/StarTreeMetadataTests.java | 4 +- .../index/mapper/ObjectMapperTests.java | 10 +- .../index/mapper/StarTreeMapperTests.java | 110 ++++++------------ 43 files changed, 575 insertions(+), 483 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/FieldValueConverter.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java index 3115dce6c10a5..b46b58f415cfd 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java @@ -49,6 +49,7 @@ import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -71,10 +72,12 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied - * by a scaling factor and rounded to the closest long. */ + * by a scaling factor and rounded to the closest long. + */ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "scaled_float"; @@ -162,11 +165,21 @@ public ScaledFloatFieldMapper build(BuilderContext context) { ); return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this); } + + @Override + public Optional getSupportedDataCubeDimensionType() { + return Optional.of(DimensionType.NUMERIC); + } + + @Override + public boolean isDataCubeMetricSupported() { + return true; + } } public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings())); - public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder { + public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder, FieldValueConverter { private final double scalingFactor; private final Double nullValue; @@ -340,6 +353,12 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { private double scale(Object input) { return new BigDecimal(Double.toString(parse(input))).multiply(BigDecimal.valueOf(scalingFactor)).doubleValue(); } + + @Override + public double toDoubleValue(long value) { + double inverseScalingFactor = 1d / scalingFactor; + return value * inverseScalingFactor; + } } private final Explicit ignoreMalformed; diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java index 45507867f6691..c3d62b088ced7 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -34,11 +34,16 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.plugins.Plugin; +import org.junit.AfterClass; +import org.junit.BeforeClass; import java.io.IOException; import java.util.Arrays; @@ -46,6 +51,7 @@ import java.util.List; import static java.util.Collections.singletonList; +import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; import static org.hamcrest.Matchers.containsString; public class ScaledFloatFieldMapperTests extends MapperTestCase { @@ -91,24 +97,112 @@ public void testExistsQueryDocValuesDisabled() throws IOException { assertParseMinimalWarnings(); } - public void testDefaults() throws Exception { - XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0)); + @BeforeClass + public static void createMapper() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build()); + } + + @AfterClass + public static void clearMapper() { + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testScaledFloatWithStarTree() throws Exception { + + double scalingFactorField1 = randomDouble() * 100; + double scalingFactorField2 = randomDouble() * 100; + double scalingFactorField3 = randomDouble() * 100; + + XContentBuilder mapping = getStarTreeMappingWithScaledFloat(scalingFactorField1, scalingFactorField2, scalingFactorField3); DocumentMapper mapper = createDocumentMapper(mapping); - assertEquals(mapping.toString(), mapper.mappingSource().toString()); + assertTrue(mapping.toString().contains("startree")); - ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123))); - IndexableField[] fields = doc.rootDoc().getFields("field"); + long randomLongField1 = randomLong(); + long randomLongField2 = randomLong(); + long randomLongField3 = randomLong(); + ParsedDocument doc = mapper.parse( + source(b -> b.field("field1", randomLongField1).field("field2", randomLongField2).field("field3", randomLongField3)) + ); + validateScaledFloatFields(doc, "field1", randomLongField1, scalingFactorField1); + validateScaledFloatFields(doc, "field2", randomLongField2, scalingFactorField2); + validateScaledFloatFields(doc, "field3", randomLongField3, scalingFactorField3); + } + + @Override + protected Settings getIndexSettings() { + return Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(super.getIndexSettings()) + .build(); + } + + private static void validateScaledFloatFields(ParsedDocument doc, String field, long value, double scalingFactor) { + IndexableField[] fields = doc.rootDoc().getFields(field); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointDimensionCount()); assertFalse(pointField.fieldType().stored()); - assertEquals(1230, pointField.numericValue().longValue()); + assertEquals((long) (value * scalingFactor), pointField.numericValue().longValue()); IndexableField dvField = fields[1]; assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); - assertEquals(1230, dvField.numericValue().longValue()); + assertEquals((long) (value * scalingFactor), dvField.numericValue().longValue()); assertFalse(dvField.fieldType().stored()); } + private XContentBuilder getStarTreeMappingWithScaledFloat( + double scalingFactorField1, + double scalingFactorField2, + double scalingFactorField3 + ) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("max_leaf_docs", 100); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "field1"); + b.endObject(); + b.startObject(); + b.field("name", "field2"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "field3"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("field1"); + b.field("type", "scaled_float").field("scaling_factor", scalingFactorField1); + b.endObject(); + b.startObject("field2"); + b.field("type", "scaled_float").field("scaling_factor", scalingFactorField2); + b.endObject(); + b.startObject("field3"); + b.field("type", "scaled_float").field("scaling_factor", scalingFactorField3); + b.endObject(); + b.endObject(); + }); + } + + public void testDefaults() throws Exception { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0)); + DocumentMapper mapper = createDocumentMapper(mapping); + assertEquals(mapping.toString(), mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123))); + validateScaledFloatFields(doc, "field", 123, 10.0); + } + public void testMissingScalingFactor() { Exception e = expectThrows( MapperParsingException.class, diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 8cfb710679137..0d69b762ab4f2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -12,7 +12,6 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -23,7 +22,6 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.compositeindex.CompositeIndexSettings; -import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -62,7 +60,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv_1") + .endObject() + .startObject() + .field("name", "numeric_dv_2") .endObject() .startObject() .field("name", getDim(invalidDim, keywordDim)) @@ -84,6 +85,14 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv_1") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric_dv_2") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -112,11 +121,7 @@ private static XContentBuilder createMaxDimTestMapping() { .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") - .startArray("calendar_intervals") - .value("day") - .value("month") - .endArray() + .field("name", "dim4") .endObject() .startObject() .field("name", "dim2") @@ -201,7 +206,7 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv1") .endObject() .startObject() .field("name", changeDim ? "numeric_new" : getDim(false, false)) @@ -223,6 +228,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv1") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -256,7 +265,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "numeric_dv2") .endObject() .startObject() .field("name", "numeric_dv") @@ -284,6 +293,10 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() + .startObject("numeric_dv2") + .field("type", "integer") + .field("doc_values", true) + .endObject() .startObject("numeric_dv1") .field("type", "integer") .field("doc_values", true) @@ -328,15 +341,8 @@ public void testValidCompositeIndex() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); - assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); @@ -496,15 +502,8 @@ public void testUpdateIndexWhenMappingIsSame() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); - assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); // Assert default metrics @@ -570,24 +569,6 @@ public void testMaxMetricsCompositeIndex() { ); } - public void testMaxCalendarIntervalsCompositeIndex() { - MapperParsingException ex = expectThrows( - MapperParsingException.class, - () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) - .setSettings( - Settings.builder() - .put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1) - .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) - ) - .get() - ); - assertEquals( - "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", - ex.getMessage() - ); - } - public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 6a09e947217f5..3c418c68fe8ad 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -12,9 +12,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; -import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.Mapper; -import org.opensearch.index.mapper.NumberFieldMapper; import java.util.ArrayList; import java.util.List; @@ -55,11 +53,13 @@ public static Dimension parseAndCreateDimension( Map dimensionMap, Mapper.TypeParser.ParserContext c ) { - if (builder instanceof DateFieldMapper.Builder) { + if (builder.getSupportedDataCubeDimensionType().isPresent() + && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.DATE)) { return parseAndCreateDateDimension(name, dimensionMap, c); - } else if (builder instanceof NumberFieldMapper.Builder) { - return new NumericDimension(name); - } + } else if (builder.getSupportedDataCubeDimensionType().isPresent() + && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.NUMERIC)) { + return new NumericDimension(name); + } throw new IllegalArgumentException( String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) ); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java new file mode 100644 index 0000000000000..4b9faea331752 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +/** + * Represents the types of dimensions supported in a data cube. + *

+ * This enum defines the possible types of dimensions that can be used + * in a data cube structure within the composite index. + * + * @opensearch.experimental + */ +public enum DimensionType { + /** + * Represents a numeric dimension type. + * This is used for dimensions that contain numerical values. + */ + NUMERIC, + + /** + * Represents a date dimension type. + * This is used for dimensions that contain date or timestamp values. + */ + DATE +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index e79abe0f170b3..2bdbab1cd5b81 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -7,22 +7,23 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; /** * Count value aggregator for star tree * * @opensearch.experimental */ -class CountValueAggregator implements ValueAggregator { +public class CountValueAggregator implements ValueAggregator { public static final long DEFAULT_INITIAL_VALUE = 1L; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; + private static final FieldValueConverter VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.LONG; public CountValueAggregator() {} @Override - public StarTreeNumericType getAggregatedValueType() { + public FieldValueConverter getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java index 0896fa54e9f46..0f354aad063c2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregator.java @@ -8,7 +8,8 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; /** * Aggregator to handle '_doc_count' field @@ -17,12 +18,12 @@ */ public class DocCountAggregator implements ValueAggregator { - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; + private static final FieldValueConverter VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.LONG; public DocCountAggregator() {} @Override - public StarTreeNumericType getAggregatedValueType() { + public FieldValueConverter getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java index 8e7bb44238e40..34a3bdc32226d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; /** * Max value aggregator for star tree @@ -16,8 +16,8 @@ */ class MaxValueAggregator extends StatelessDoubleValueAggregator { - public MaxValueAggregator(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType, null); + public MaxValueAggregator(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter, null); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index 8b6db2a183bf8..487e714c9e44e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -8,8 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.FieldValueConverter; import java.util.Comparator; import java.util.Objects; @@ -27,15 +26,15 @@ public class MetricAggregatorInfo implements Comparable { private final MetricStat metricStat; private final String field; private final ValueAggregator valueAggregators; - private final StarTreeNumericType starTreeNumericType; + private final FieldValueConverter fieldValueConverter; /** * Constructor for MetricAggregatorInfo */ - public MetricAggregatorInfo(MetricStat metricStat, String field, String starFieldName, IndexNumericFieldData.NumericType numericType) { + public MetricAggregatorInfo(MetricStat metricStat, String field, String starFieldName, FieldValueConverter fieldValueConverter) { this.metricStat = metricStat; - this.starTreeNumericType = StarTreeNumericType.fromNumericType(numericType); - this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat, this.starTreeNumericType); + this.fieldValueConverter = fieldValueConverter; + this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat, this.fieldValueConverter); this.field = field; this.starFieldName = starFieldName; this.metric = toFieldName(); @@ -72,8 +71,8 @@ public ValueAggregator getValueAggregators() { /** * @return star tree aggregated value type */ - public StarTreeNumericType getAggregatedValueType() { - return starTreeNumericType; + public FieldValueConverter getNumericFieldConverter() { + return fieldValueConverter; } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java index 46e9188b5dc2f..b085447be758e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; /** * Min value aggregator for star tree @@ -16,8 +16,8 @@ */ class MinValueAggregator extends StatelessDoubleValueAggregator { - public MinValueAggregator(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType, null); + public MinValueAggregator(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter, null); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java index 30a1c47c0ee9b..d83a828fb0f9c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StatelessDoubleValueAggregator.java @@ -7,7 +7,8 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; /** * This is an abstract class that defines the common methods for all double value aggregators @@ -17,17 +18,17 @@ */ abstract class StatelessDoubleValueAggregator implements ValueAggregator { - protected final StarTreeNumericType starTreeNumericType; + protected final FieldValueConverter fieldValueConverter; protected final Double identityValue; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private static final FieldValueConverter VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.DOUBLE; - public StatelessDoubleValueAggregator(StarTreeNumericType starTreeNumericType, Double identityValue) { - this.starTreeNumericType = starTreeNumericType; + public StatelessDoubleValueAggregator(FieldValueConverter fieldValueConverter, Double identityValue) { + this.fieldValueConverter = fieldValueConverter; this.identityValue = identityValue; } @Override - public StarTreeNumericType getAggregatedValueType() { + public FieldValueConverter getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } @@ -36,7 +37,7 @@ public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue) if (segmentDocValue == null) { return getIdentityMetricValue(); } - return starTreeNumericType.getDoubleValue(segmentDocValue); + return fieldValueConverter.toDoubleValue(segmentDocValue); } @Override @@ -57,7 +58,7 @@ public Double toAggregatedValueType(Long value) { if (value == null) { return getIdentityMetricValue(); } - return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index ef97a9b603df3..81eb3558915be 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -7,7 +7,8 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.metrics.CompensatedSum; /** @@ -22,17 +23,17 @@ */ class SumValueAggregator implements ValueAggregator { - private final StarTreeNumericType starTreeNumericType; - private static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private final FieldValueConverter fieldValueConverter; + private static final FieldValueConverter VALUE_AGGREGATOR_TYPE = NumberFieldMapper.NumberType.DOUBLE; private CompensatedSum kahanSummation = new CompensatedSum(0, 0); - public SumValueAggregator(StarTreeNumericType starTreeNumericType) { - this.starTreeNumericType = starTreeNumericType; + public SumValueAggregator(FieldValueConverter fieldValueConverter) { + this.fieldValueConverter = fieldValueConverter; } @Override - public StarTreeNumericType getAggregatedValueType() { + public FieldValueConverter getAggregatedValueType() { return VALUE_AGGREGATOR_TYPE; } @@ -41,7 +42,7 @@ public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue) kahanSummation.reset(0, 0); // add takes care of the sum and compensation internally if (segmentDocValue != null) { - kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + kahanSummation.add(fieldValueConverter.toDoubleValue(segmentDocValue)); } else { kahanSummation.add(getIdentityMetricValue()); } @@ -55,7 +56,7 @@ public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocV assert value == null || kahanSummation.value() == value; // add takes care of the sum and compensation internally if (segmentDocValue != null) { - kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + kahanSummation.add(fieldValueConverter.toDoubleValue(segmentDocValue)); } else { kahanSummation.add(getIdentityMetricValue()); } @@ -92,7 +93,7 @@ public Double toAggregatedValueType(Long value) { if (value == null) { return getIdentityMetricValue(); } - return VALUE_AGGREGATOR_TYPE.getDoubleValue(value); + return VALUE_AGGREGATOR_TYPE.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index d5ca7f3493087..8eac45669c063 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -7,7 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. @@ -19,7 +19,7 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - StarTreeNumericType getAggregatedValueType(); + FieldValueConverter getAggregatedValueType(); /** * Returns the initial aggregated value. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index bdc381110365d..aac5d2b2855d3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; /** * Value aggregator factory for a given aggregation type @@ -22,20 +22,20 @@ private ValueAggregatorFactory() {} * Returns a new instance of value aggregator for the given aggregation type. * * @param aggregationType Aggregation type - * @param starTreeNumericType Numeric type associated with star tree field ( as specified in index mapping ) + * @param fieldValueConverter Numeric type converter associated with star tree field ( as specified in index mapping ) * @return Value aggregator */ - public static ValueAggregator getValueAggregator(MetricStat aggregationType, StarTreeNumericType starTreeNumericType) { + public static ValueAggregator getValueAggregator(MetricStat aggregationType, FieldValueConverter fieldValueConverter) { switch (aggregationType) { // avg aggregator will be covered in the part of query (using count and sum) case SUM: - return new SumValueAggregator(starTreeNumericType); + return new SumValueAggregator(fieldValueConverter); case VALUE_COUNT: return new CountValueAggregator(); case MIN: - return new MinValueAggregator(starTreeNumericType); + return new MinValueAggregator(fieldValueConverter); case MAX: - return new MaxValueAggregator(starTreeNumericType); + return new MaxValueAggregator(fieldValueConverter); case DOC_COUNT: return new DocCountAggregator(); default: diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java deleted file mode 100644 index 57fe573a6a93c..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; - -import org.opensearch.index.fielddata.IndexNumericFieldData; - -import java.util.function.Function; - -/** - * Enum to map Star Tree Numeric Types to Lucene's Numeric Type - * - * @opensearch.experimental - */ -public enum StarTreeNumericType { - - // TODO: Handle scaled floats - HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), - FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), - LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), - DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), - INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), - SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), - BYTE(IndexNumericFieldData.NumericType.BYTE, StarTreeNumericTypeConverters::bytesToDouble), - UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); - - final IndexNumericFieldData.NumericType numericType; - final Function converter; - - StarTreeNumericType(IndexNumericFieldData.NumericType numericType, Function converter) { - this.numericType = numericType; - this.converter = converter; - } - - public double getDoubleValue(long rawValue) { - return this.converter.apply(rawValue); - } - - public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericType numericType) { - switch (numericType) { - case HALF_FLOAT: - return StarTreeNumericType.HALF_FLOAT; - case FLOAT: - return StarTreeNumericType.FLOAT; - case LONG: - return StarTreeNumericType.LONG; - case DOUBLE: - return StarTreeNumericType.DOUBLE; - case INT: - return StarTreeNumericType.INT; - case SHORT: - return StarTreeNumericType.SHORT; - case UNSIGNED_LONG: - return StarTreeNumericType.UNSIGNED_LONG; - case BYTE: - return StarTreeNumericType.BYTE; - default: - throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java deleted file mode 100644 index eb7647c4f9851..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; - -import org.apache.lucene.sandbox.document.HalfFloatPoint; -import org.apache.lucene.util.NumericUtils; -import org.opensearch.common.Numbers; -import org.opensearch.common.annotation.ExperimentalApi; - -/** - * Numeric converters used during aggregations of metric values - * - * @opensearch.experimental - */ -@ExperimentalApi -public class StarTreeNumericTypeConverters { - - public static double halfFloatPointToDouble(Long value) { - return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue()); - } - - public static double floatPointToDouble(Long value) { - return NumericUtils.sortableIntToFloat((int) value.longValue()); - } - - public static double longToDouble(Long value) { - return (double) value; - } - - public static double intToDouble(Long value) { - return (double) value; - } - - public static double shortToDouble(Long value) { - return (double) value; - } - - public static Double sortableLongtoDouble(Long value) { - return NumericUtils.sortableLongToDouble(value); - } - - public static double unsignedlongToDouble(Long value) { - return Numbers.unsignedLongToDouble(value); - } - - public static double bytesToDouble(Long value) { - byte[] bytes = new byte[8]; - NumericUtils.longToSortableBytes(value, bytes, 0); - return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); - } - -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java deleted file mode 100644 index fe5c2a7ceb254..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Numeric Types for Composite Index Star Tree - * - * @opensearch.experimental - */ -package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java index 4214a46b2fc1c..327fd26c00608 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java @@ -19,13 +19,16 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericTypeConverters; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeDocumentBitSetUtil; +import org.opensearch.index.mapper.FieldValueConverter; import java.io.Closeable; import java.io.IOException; import java.util.List; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.DOUBLE; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; + /** * Abstract class for managing star tree file operations. * @@ -89,24 +92,22 @@ protected int writeDimensions(StarTreeDocument starTreeDocument, IndexOutput out protected int writeMetrics(StarTreeDocument starTreeDocument, IndexOutput output, boolean isAggregatedDoc) throws IOException { int numBytes = 0; for (int i = 0; i < starTreeDocument.metrics.length; i++) { - switch (metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType()) { - case LONG: + FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); + if (aggregatedValueType.equals(LONG)) { + output.writeLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); + numBytes += Long.BYTES; + } else if (aggregatedValueType.equals(DOUBLE)) { + if (isAggregatedDoc) { + long val = NumericUtils.doubleToSortableLong( + starTreeDocument.metrics[i] == null ? 0.0 : (Double) starTreeDocument.metrics[i] + ); + output.writeLong(val); + } else { output.writeLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); - numBytes += Long.BYTES; - break; - case DOUBLE: - if (isAggregatedDoc) { - long val = NumericUtils.doubleToSortableLong( - starTreeDocument.metrics[i] == null ? 0.0 : (Double) starTreeDocument.metrics[i] - ); - output.writeLong(val); - } else { - output.writeLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); - } - numBytes += Long.BYTES; - break; - default: - throw new IllegalStateException("Unsupported metric type"); + } + numBytes += Long.BYTES; + } else { + throw new IllegalStateException("Unsupported metric type"); } } numBytes += StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.metrics, output); @@ -157,22 +158,20 @@ protected long readDimensions(Long[] dimensions, RandomAccessInput input, long o protected long readMetrics(RandomAccessInput input, long offset, int numMetrics, Object[] metrics, boolean isAggregatedDoc) throws IOException { for (int i = 0; i < numMetrics; i++) { - switch (metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType()) { - case LONG: - metrics[i] = input.readLong(offset); - offset += Long.BYTES; - break; - case DOUBLE: - long val = input.readLong(offset); - if (isAggregatedDoc) { - metrics[i] = StarTreeNumericTypeConverters.sortableLongtoDouble(val); - } else { - metrics[i] = val; - } - offset += Long.BYTES; - break; - default: - throw new IllegalStateException("Unsupported metric type"); + FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); + if (aggregatedValueType.equals(LONG)) { + metrics[i] = input.readLong(offset); + offset += Long.BYTES; + } else if (aggregatedValueType.equals(DOUBLE)) { + long val = input.readLong(offset); + if (isAggregatedDoc) { + metrics[i] = DOUBLE.toDoubleValue(val); + } else { + metrics[i] = val; + } + offset += Long.BYTES; + } else { + throw new IllegalStateException("Unsupported metric type"); } } offset += StarTreeDocumentBitSetUtil.readBitSet(input, offset, metrics, index -> null); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 13d5059a241b1..ba4cb792c00df 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -34,11 +34,11 @@ import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.FieldMapper; +import org.opensearch.index.mapper.FieldValueConverter; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.NumberFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -55,6 +55,8 @@ import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.getFieldInfo; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.DOUBLE; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; /** * Builder for star tree. Defines the algorithm to construct star-tree @@ -143,7 +145,7 @@ public List generateMetricAggregatorInfos(MapperService ma MetricStat.DOC_COUNT, metric.getField(), starTreeField.getName(), - IndexNumericFieldData.NumericType.LONG + LONG ); metricAggregatorInfos.add(metricAggregatorInfo); continue; @@ -152,10 +154,10 @@ public List generateMetricAggregatorInfos(MapperService ma if (metricStat.isDerivedMetric()) { continue; } - IndexNumericFieldData.NumericType numericType; + FieldValueConverter fieldValueConverter; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); - if (fieldMapper instanceof NumberFieldMapper) { - numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + if (fieldMapper instanceof FieldMapper && ((FieldMapper) fieldMapper).fieldType() instanceof FieldValueConverter) { + fieldValueConverter = (FieldValueConverter) ((FieldMapper) fieldMapper).fieldType(); } else { logger.error("unsupported mapper type"); throw new IllegalStateException("unsupported mapper type"); @@ -165,7 +167,7 @@ public List generateMetricAggregatorInfos(MapperService ma metricStat, metric.getField(), starTreeField.getName(), - numericType + fieldValueConverter ); metricAggregatorInfos.add(metricAggregatorInfo); } @@ -351,20 +353,17 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A for (int i = 0; i < starTreeDocument.metrics.length; i++) { try { - switch (metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType()) { - case LONG: - if (starTreeDocument.metrics[i] != null) { - metricWriters.get(i).addValue(docId, (long) starTreeDocument.metrics[i]); - } - break; - case DOUBLE: - if (starTreeDocument.metrics[i] != null) { - metricWriters.get(i) - .addValue(docId, NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i])); - } - break; - default: - throw new IllegalStateException("Unknown metric doc value type"); + FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); + if (aggregatedValueType.equals(LONG)) { + if (starTreeDocument.metrics[i] != null) { + metricWriters.get(i).addValue(docId, (long) starTreeDocument.metrics[i]); + } + } else if (aggregatedValueType.equals(DOUBLE)) { + if (starTreeDocument.metrics[i] != null) { + metricWriters.get(i).addValue(docId, NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i])); + } + } else { + throw new IllegalStateException("Unknown metric doc value type"); } } catch (IllegalArgumentException e) { logger.error("could not parse the value, exiting creation of star tree"); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index 0ae704afa53b9..df2ce9096bfc1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -250,7 +250,7 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IO } int high = getInt(LAST_CHILD_ID_OFFSET); - // if the current node is null node, increment the low to reduce the search space + // if the current node is null node, decrement the high to reduce the search space if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, high), StarTreeNodeType.NULL) != null) { high--; } diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index e60d5a739a9ae..bc50f507010c4 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -349,6 +349,7 @@ public DateFieldMapper build(BuilderContext context) { Long nullTimestamp = parseNullValue(ft); return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } + } public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> { diff --git a/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java index 0a461fff38fac..db4770adf6666 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java @@ -75,6 +75,11 @@ protected List> getParameters() { public DocCountFieldMapper build(BuilderContext context) { return new DocCountFieldMapper(); } + + @Override + public boolean isDataCubeMetricSupported() { + return true; + } } /** diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldValueConverter.java b/server/src/main/java/org/opensearch/index/mapper/FieldValueConverter.java new file mode 100644 index 0000000000000..cb16dba6ab640 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/FieldValueConverter.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +/** + *

+ * Implementations of this interface should define the conversion logic + * from a sortable long value to a double value, taking into account any necessary + * scaling, normalization, or other transformations required by the specific + * field type. + * + * @opensearch.experimental + */ +public interface FieldValueConverter { + + /** + * Converts the Lucene representation of the value as a long to an actual double representation + * + * @param value the long value to be converted + * @return the corresponding double value + */ + double toDoubleValue(long value); + +} diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index 46a5050d4fc18..87fdd8266a795 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -100,7 +100,7 @@ public Version indexCreatedVersionOrDefault(@Nullable Version defaultValue) { * @opensearch.api */ @PublicApi(since = "1.0.0") - public abstract static class Builder { + public abstract static class Builder implements MapperBuilderProperties { public String name; diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java new file mode 100644 index 0000000000000..ce7b8f28b0b29 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/MapperBuilderProperties.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.index.compositeindex.datacube.DimensionType; + +import java.util.Optional; + +/** + * An interface that defines properties for MapperBuilder implementations. + * + * @opensearch.experimental + */ +public interface MapperBuilderProperties { + + /** + * Indicates whether the implementation supports data cube dimensions. + * + * @return an Optional containing the supported DimensionType if data cube dimensions are supported, + * or an empty Optional if not supported + */ + default Optional getSupportedDataCubeDimensionType() { + return Optional.empty(); + } + + /** + * Indicates whether the implementation supports data cube metrics. + * + * @return true if data cube metrics are supported, false otherwise + */ + default boolean isDataCubeMetricSupported() { + return false; + } + +} diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 9286b5c64b5f2..eb0694edc70ba 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -63,6 +63,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.document.SortedUnsignedLongDocValuesRangeQuery; import org.opensearch.index.document.SortedUnsignedLongDocValuesSetQuery; import org.opensearch.index.fielddata.IndexFieldData; @@ -84,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -172,6 +174,22 @@ public NumberFieldMapper build(BuilderContext context) { MappedFieldType ft = new NumberFieldType(buildFullName(context), this); return new NumberFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } + + @Override + public Optional getSupportedDataCubeDimensionType() { + + // unsigned long is not supported as dimension for star tree + if (type.numericType.equals(NumericType.UNSIGNED_LONG)) { + return Optional.empty(); + } + + return Optional.of(DimensionType.NUMERIC); + } + + @Override + public boolean isDataCubeMetricSupported() { + return true; + } } /** @@ -179,7 +197,7 @@ public NumberFieldMapper build(BuilderContext context) { * * @opensearch.internal */ - public enum NumberType implements NumericPointEncoder { + public enum NumberType implements NumericPointEncoder, FieldValueConverter { HALF_FLOAT("half_float", NumericType.HALF_FLOAT) { @Override public Float parse(Object value, boolean coerce) { @@ -209,6 +227,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return HalfFloatPoint.sortableShortToHalfFloat((short) value); + } + @Override public Float parse(XContentParser parser, boolean coerce) throws IOException { float parsed = parser.floatValue(coerce); @@ -353,6 +376,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return NumericUtils.sortableIntToFloat((int) value); + } + @Override public Float parse(XContentParser parser, boolean coerce) throws IOException { float parsed = parser.floatValue(coerce); @@ -486,6 +514,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return NumericUtils.sortableLongToDouble(value); + } + @Override public Double parse(XContentParser parser, boolean coerce) throws IOException { double parsed = parser.doubleValue(coerce); @@ -618,6 +651,13 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + byte[] bytes = new byte[8]; + NumericUtils.longToSortableBytes(value, bytes, 0); + return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); + } + @Override public Short parse(XContentParser parser, boolean coerce) throws IOException { int value = parser.intValue(coerce); @@ -697,6 +737,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return (double) value; + } + @Override public Short parse(XContentParser parser, boolean coerce) throws IOException { return parser.shortValue(coerce); @@ -772,6 +817,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return (double) value; + } + @Override public Integer parse(XContentParser parser, boolean coerce) throws IOException { return parser.intValue(coerce); @@ -943,6 +993,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return (double) value; + } + @Override public Long parse(XContentParser parser, boolean coerce) throws IOException { return parser.longValue(coerce); @@ -1070,6 +1125,11 @@ public byte[] encodePoint(Number value) { return point; } + @Override + public double toDoubleValue(long value) { + return Numbers.unsignedLongToDouble(value); + } + @Override public BigInteger parse(XContentParser parser, boolean coerce) throws IOException { return parser.bigIntegerValue(coerce); @@ -1182,12 +1242,16 @@ Number valueForSearch(String value) { this.parser = new TypeParser((n, c) -> new Builder(n, this, c.getSettings())); } - /** Get the associated type name. */ + /** + * Get the associated type name. + */ public final String typeName() { return name; } - /** Get the associated numeric type */ + /** + * Get the associated numeric type + */ public final NumericType numericType() { return numericType; } @@ -1486,7 +1550,7 @@ protected String toString(byte[] value) { * * @opensearch.internal */ - public static class NumberFieldType extends SimpleMappedFieldType implements NumericPointEncoder { + public static class NumberFieldType extends SimpleMappedFieldType implements NumericPointEncoder, FieldValueConverter { private final NumberType type; private final boolean coerce; @@ -1652,6 +1716,11 @@ public Number parsePoint(byte[] value) { public byte[] encodePoint(Number value) { return type.encodePoint(value); } + + @Override + public double toDoubleValue(long value) { + return type.toDoubleValue(value); + } } private final NumberType type; diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index 17c27ef149e54..9e2e7ef3c3889 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -62,11 +62,6 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { */ public static class Builder extends ParametrizedFieldMapper.Builder { private ObjectMapper.Builder objbuilder; - private static final Set> ALLOWED_DIMENSION_MAPPER_BUILDERS = Set.of( - NumberFieldMapper.Builder.class, - DateFieldMapper.Builder.class - ); - private static final Set> ALLOWED_METRIC_MAPPER_BUILDERS = Set.of(NumberFieldMapper.Builder.class); @SuppressWarnings("unchecked") private final Parameter config = new Parameter<>(CONFIG, false, () -> null, (name, context, nodeObj) -> { @@ -380,11 +375,11 @@ protected List> getParameters() { } private static boolean isBuilderAllowedForDimension(Mapper.Builder builder) { - return ALLOWED_DIMENSION_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + return builder.getSupportedDataCubeDimensionType().isPresent(); } private static boolean isBuilderAllowedForMetric(Mapper.Builder builder) { - return ALLOWED_METRIC_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + return builder.isDataCubeMetricSupported(); } private Optional findMapperBuilderByName(String field, List mappersBuilders) { diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index 4bbefeba0845b..0c6d21d28cc8a 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -46,9 +46,9 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.indices.IndicesModule; import org.junit.After; import org.junit.AfterClass; @@ -159,7 +159,7 @@ public void testStarTreeDocValues() throws IOException { StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( List.of(starTreeValues), - List.of(StarTreeNumericType.DOUBLE, StarTreeNumericType.LONG, StarTreeNumericType.LONG), + List.of(NumberFieldMapper.NumberType.DOUBLE, NumberFieldMapper.NumberType.LONG, NumberFieldMapper.NumberType.LONG), reader.maxDoc() ); assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 7b5672529dc8d..f3e111bf6caa6 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -12,7 +12,6 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -21,6 +20,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.FieldValueConverter; import java.io.IOException; import java.util.ArrayDeque; @@ -44,7 +44,7 @@ public class StarTreeTestUtils { public static StarTreeDocument[] getSegmentsStarTreeDocuments( List starTreeValuesSubs, - List starTreeNumericTypes, + List fieldValueConverters, int numDocs ) throws IOException { List starTreeDocuments = new ArrayList<>(); @@ -72,7 +72,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( } int currentDocId = 0; while (currentDocId < numDocs) { - starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, starTreeNumericTypes)); + starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, fieldValueConverters)); currentDocId++; } } @@ -84,7 +84,7 @@ public static StarTreeDocument getStarTreeDocument( int currentDocId, SequentialDocValuesIterator[] dimensionReaders, List metricReaders, - List starTreeNumericTypes + List fieldValueConverters ) throws IOException { Long[] dims = new Long[dimensionReaders.length]; int i = 0; @@ -98,15 +98,15 @@ public static StarTreeDocument getStarTreeDocument( Object[] metrics = new Object[metricReaders.size()]; for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { metricDocValuesIterator.nextDoc(currentDocId); - metrics[i] = toStarTreeNumericTypeValue(metricDocValuesIterator.value(currentDocId), starTreeNumericTypes.get(i)); + metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i)); i++; } return new StarTreeDocument(dims, metrics); } - public static Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType starTreeNumericType) { + public static Double toAggregatorValueType(Long value, FieldValueConverter fieldValueConverter) { try { - return starTreeNumericType.getDoubleValue(value); + return fieldValueConverter.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java index 36f75834abba8..00f9fcf0e269f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/AbstractValueAggregatorTests.java @@ -10,7 +10,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -21,27 +22,27 @@ public abstract class AbstractValueAggregatorTests extends OpenSearchTestCase { private ValueAggregator aggregator; - protected StarTreeNumericType starTreeNumericType; + protected FieldValueConverter fieldValueConverter; - public AbstractValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - this.starTreeNumericType = starTreeNumericType; + public AbstractValueAggregatorTests(FieldValueConverter fieldValueConverter) { + this.fieldValueConverter = fieldValueConverter; } @Before public void setup() { - aggregator = getValueAggregator(starTreeNumericType); + aggregator = getValueAggregator(fieldValueConverter); } @ParametersFactory public static Collection parameters() { List parameters = new ArrayList<>(); - for (StarTreeNumericType starTreeNumericType : StarTreeNumericType.values()) { - parameters.add(new Object[] { starTreeNumericType }); + for (FieldValueConverter fieldValueConverter : NumberFieldMapper.NumberType.values()) { + parameters.add(new Object[] { fieldValueConverter }); } return parameters; } - public abstract ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType); + public abstract ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter); public void testGetInitialAggregatedValueForSegmentDocNullValue() { assertEquals(aggregator.getIdentityMetricValue(), aggregator.getInitialAggregatedValueForSegmentDocValue(null)); @@ -61,6 +62,10 @@ public void testGetInitialAggregatedNullValue() { public void testGetInitialAggregatedValueForSegmentDocValue() { long randomLong = randomLong(); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong)); + if (aggregator instanceof CountValueAggregator) { + assertEquals(CountValueAggregator.DEFAULT_INITIAL_VALUE, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong())); + } else { + assertEquals(fieldValueConverter.toDoubleValue(randomLong), aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong)); + } } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java index b270c1b1bc26c..db22ee9af18e2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -8,14 +8,14 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; public class CountValueAggregatorTests extends AbstractValueAggregatorTests { private CountValueAggregator aggregator; - public CountValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public CountValueAggregatorTests(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter); } public void testMergeAggregatedValueAndSegmentValue() { @@ -52,7 +52,7 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { + public ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter) { aggregator = new CountValueAggregator(); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java index 2765629aa5950..3960412d65868 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/DocCountAggregatorTests.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; /** * Unit tests for {@link DocCountAggregator}. @@ -17,8 +17,8 @@ public class DocCountAggregatorTests extends AbstractValueAggregatorTests { private DocCountAggregator aggregator; - public DocCountAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public DocCountAggregatorTests(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter); } public void testMergeAggregatedValueAndSegmentValue() { @@ -52,7 +52,7 @@ public void testGetInitialAggregatedValue() { assertEquals(randomLong, (long) aggregator.getInitialAggregatedValue(randomLong)); } - public void testToStarTreeNumericTypeValue() { + public void testToAggregatedValueType() { long randomLong = randomLong(); assertEquals(randomLong, (long) aggregator.toAggregatedValueType(randomLong)); } @@ -62,7 +62,7 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { + public ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter) { aggregator = new DocCountAggregator(); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java index b103416251c46..b713a47ed5f62 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MaxValueAggregatorTests.java @@ -9,27 +9,28 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; public class MaxValueAggregatorTests extends AbstractValueAggregatorTests { private MaxValueAggregator aggregator; - public MaxValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public MaxValueAggregatorTests(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter); } public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.max(starTreeNumericType.getDoubleValue(randomLong), randomDouble), + Math.max(fieldValueConverter.toDoubleValue(randomLong), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals(fieldValueConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); - assertEquals(Math.max(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); + assertEquals(Math.max(2.0, fieldValueConverter.toDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); } public void testMergeAggregatedValues() { @@ -46,7 +47,7 @@ public void testGetInitialAggregatedValue() { } public void testToAggregatedValueType() { - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); long randomLong = randomLong(); assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0); } @@ -56,8 +57,8 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new MaxValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter) { + aggregator = new MaxValueAggregator(fieldValueConverter); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 62671ffa03b82..5588cc1b366c9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; public class MetricAggregatorInfoTests extends OpenSearchTestCase { @@ -19,7 +19,7 @@ public void testConstructor() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(MetricStat.SUM, pair.getMetricStat()); assertEquals("column1", pair.getField()); @@ -30,7 +30,7 @@ public void testCountStarConstructor() { MetricStat.VALUE_COUNT, "anything", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(MetricStat.VALUE_COUNT, pair.getMetricStat()); assertEquals("anything", pair.getField()); @@ -41,7 +41,7 @@ public void testToFieldName() { MetricStat.SUM, "column2", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals("star_tree_field_column2_sum", pair.toFieldName()); } @@ -51,23 +51,20 @@ public void testEquals() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) - ); - assertNotEquals( - pair1, - new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) + new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", NumberFieldMapper.NumberType.DOUBLE) ); + assertNotEquals(pair1, new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", NumberFieldMapper.NumberType.DOUBLE)); } public void testHashCode() { @@ -75,13 +72,13 @@ public void testHashCode() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertEquals(pair1.hashCode(), pair2.hashCode()); } @@ -91,19 +88,19 @@ public void testCompareTo() { MetricStat.SUM, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column2", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); MetricAggregatorInfo pair3 = new MetricAggregatorInfo( MetricStat.VALUE_COUNT, "column1", "star_tree_field", - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); assertTrue(pair1.compareTo(pair2) < 0); assertTrue(pair2.compareTo(pair1) > 0); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java index 013c60d8a1b91..ec406d71ff633 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MinValueAggregatorTests.java @@ -9,26 +9,27 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; +import org.opensearch.index.mapper.NumberFieldMapper; public class MinValueAggregatorTests extends AbstractValueAggregatorTests { private MinValueAggregator aggregator; - public MinValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public MinValueAggregatorTests(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter); } public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); double randomDouble = randomDouble(); assertEquals( - Math.min(starTreeNumericType.getDoubleValue(randomLong), randomDouble), + Math.min(fieldValueConverter.toDoubleValue(randomLong), randomDouble), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals(fieldValueConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); assertEquals(randomDouble, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, null), 0.0); - assertEquals(Math.min(2.0, starTreeNumericType.getDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); + assertEquals(Math.min(2.0, fieldValueConverter.toDoubleValue(3L)), aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L), 0.0); } public void testMergeAggregatedValues() { @@ -45,7 +46,7 @@ public void testGetInitialAggregatedValue() { } public void testToAggregatedValueType() { - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); long randomLong = randomLong(); assertEquals(NumericUtils.sortableLongToDouble(randomLong), aggregator.toAggregatedValueType(randomLong), 0.0); } @@ -55,8 +56,8 @@ public void testIdentityMetricValue() { } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new MinValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter) { + aggregator = new MinValueAggregator(fieldValueConverter); return aggregator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java index 487bd54602503..2ba29a1165dc8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/StaticValueAggregatorTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.metrics.CompensatedSum; import org.opensearch.test.OpenSearchTestCase; @@ -39,7 +39,7 @@ public void testKahanSummation() { private static double getAggregatedValue(double[] numbers) { // explicitly took double to test for most precision // hard to run similar tests for different data types dynamically as inputs and precision vary - SumValueAggregator aggregator = new SumValueAggregator(StarTreeNumericType.DOUBLE); + SumValueAggregator aggregator = new SumValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue(aggregatedValue, NumericUtils.doubleToSortableLong(numbers[1])); aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue(aggregatedValue, NumericUtils.doubleToSortableLong(numbers[2])); @@ -65,7 +65,7 @@ private double normalSum(double[] numbers) { public void testMaxAggregatorExtremeValues() { double[] numbers = { Double.MAX_VALUE, Double.MIN_VALUE, 0.0, Double.MAX_VALUE + 1 }; double expected = Double.MAX_VALUE + 1; - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -85,7 +85,7 @@ public void testMaxAggregatorExtremeValues_Infinity() { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; double expected = Double.POSITIVE_INFINITY; - MaxValueAggregator aggregator = new MaxValueAggregator(StarTreeNumericType.DOUBLE); + MaxValueAggregator aggregator = new MaxValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -99,7 +99,7 @@ public void testMaxAggregatorExtremeValues_Infinity() { public void testMinAggregatorExtremeValues() { double[] numbers = { Double.MAX_VALUE, Double.MIN_VALUE - 1, 0.0, Double.MAX_VALUE + 1 }; double expected = Double.MIN_VALUE - 1; - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( @@ -119,7 +119,7 @@ public void testMinAggregatorExtremeValues_Infinity() { Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; double expected = Double.NEGATIVE_INFINITY; - MinValueAggregator aggregator = new MinValueAggregator(StarTreeNumericType.DOUBLE); + MinValueAggregator aggregator = new MinValueAggregator(NumberFieldMapper.NumberType.DOUBLE); double aggregatedValue = aggregator.getInitialAggregatedValueForSegmentDocValue(NumericUtils.doubleToSortableLong(numbers[0])); for (int i = 1; i < numbers.length; i++) { aggregatedValue = aggregator.mergeAggregatedValueAndSegmentValue( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 44c7f17a276b4..ef328fe7eafc2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -8,19 +8,19 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.FieldValueConverter; public class SumValueAggregatorTests extends AbstractValueAggregatorTests { private SumValueAggregator aggregator; - public SumValueAggregatorTests(StarTreeNumericType starTreeNumericType) { - super(starTreeNumericType); + public SumValueAggregatorTests(FieldValueConverter fieldValueConverter) { + super(fieldValueConverter); } @Override - public ValueAggregator getValueAggregator(StarTreeNumericType starTreeNumericType) { - aggregator = new SumValueAggregator(starTreeNumericType); + public ValueAggregator getValueAggregator(FieldValueConverter fieldValueConverter) { + aggregator = new SumValueAggregator(fieldValueConverter); return aggregator; } @@ -29,7 +29,7 @@ public void testMergeAggregatedValueAndSegmentValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(randomDouble); assertEquals( - randomDouble + starTreeNumericType.getDoubleValue(randomLong), + randomDouble + fieldValueConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble, randomLong), 0.0 ); @@ -41,7 +41,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { aggregator.getInitialAggregatedValue(randomDouble1); assertEquals(randomDouble1, aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, null), 0.0); assertEquals( - randomDouble1 + starTreeNumericType.getDoubleValue(randomLong), + randomDouble1 + fieldValueConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(randomDouble1, randomLong), 0.0 ); @@ -50,7 +50,7 @@ public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { public void testMergeAggregatedValueAndSegmentValue_nullInitialDocValue() { Long randomLong = randomLong(); aggregator.getInitialAggregatedValue(null); - assertEquals(starTreeNumericType.getDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); + assertEquals(fieldValueConverter.toDoubleValue(randomLong), aggregator.mergeAggregatedValueAndSegmentValue(null, randomLong), 0.0); } public void testMergeAggregatedValues() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index 6572d75d5b738..96ab1b3a52e8b 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -9,31 +9,31 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; public class ValueAggregatorFactoryTests extends OpenSearchTestCase { public void testGetValueAggregatorForSumType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(SumValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForMinType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MIN, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MIN, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(MinValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForMaxType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MAX, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.MAX, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(MaxValueAggregator.class, aggregator.getClass()); } public void testGetValueAggregatorForCountType() { - ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.VALUE_COUNT, StarTreeNumericType.LONG); + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.VALUE_COUNT, NumberFieldMapper.NumberType.LONG); assertNotNull(aggregator); assertEquals(CountValueAggregator.class, aggregator.getClass()); } @@ -41,7 +41,7 @@ public void testGetValueAggregatorForCountType() { public void testGetValueAggregatorForAvgType() { assertThrows( IllegalStateException.class, - () -> ValueAggregatorFactory.getValueAggregator(MetricStat.AVG, StarTreeNumericType.LONG) + () -> ValueAggregatorFactory.getValueAggregator(MetricStat.AVG, NumberFieldMapper.NumberType.LONG) ); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 6e23e5eacaecd..ad54bda9a916e 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -47,7 +47,6 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; @@ -56,6 +55,7 @@ import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.FieldValueConverter; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MappingLookup; @@ -1895,13 +1895,13 @@ private void validateStarTreeFileFormats( StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), starTreeValues.getStarTreeDocumentCount()); - List starTreeNumericTypes = new ArrayList<>(); + List fieldValueConverters = new ArrayList<>(); builder.metricAggregatorInfos.forEach( - metricAggregatorInfo -> starTreeNumericTypes.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) + metricAggregatorInfo -> fieldValueConverters.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) ); StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( List.of(starTreeValues), - starTreeNumericTypes, + fieldValueConverters, readState.segmentInfo.maxDoc() ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 22a0fb7875714..8f750b09c75ee 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -36,7 +36,6 @@ import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -229,8 +228,8 @@ Iterator mergeStarTrees(List starTreeValues) t public void test_generateMetricAggregatorInfos() throws IOException { List metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService); List expectedMetricAggregatorInfos = List.of( - new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE), - new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE) + new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), NumberFieldMapper.NumberType.DOUBLE), + new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), NumberFieldMapper.NumberType.DOUBLE) ); assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos); } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index 6fac5362c3dbb..2127eeb9636d3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -31,8 +31,8 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; -import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -137,7 +137,7 @@ public void test_starTreeMetadata() throws IOException { metricType, metric.getField(), starTreeField.getName(), - IndexNumericFieldData.NumericType.DOUBLE + NumberFieldMapper.NumberType.DOUBLE ); metricAggregatorInfos.add(metricAggregatorInfo); } diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index ff0533de4fe8d..cb06bf23d9cbe 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -504,7 +504,7 @@ public void testCompositeFields() throws Exception { .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "@timestamp") + .field("name", "node") .endObject() .startObject() .field("name", "status") @@ -522,8 +522,8 @@ public void testCompositeFields() throws Exception { .endObject() .endObject() .startObject("properties") - .startObject("@timestamp") - .field("type", "date") + .startObject("node") + .field("type", "integer") .endObject() .startObject("status") .field("type", "integer") @@ -561,9 +561,9 @@ public void testCompositeFields() throws Exception { StarTreeMapper starTreeMapper = (StarTreeMapper) mapper; assertEquals("star_tree", starTreeMapper.fieldType().typeName()); // Check that field in properties was parsed correctly as well - mapper = documentMapper.root().getMapper("@timestamp"); + mapper = documentMapper.root().getMapper("node"); assertNotNull(mapper); - assertEquals("date", mapper.typeName()); + assertEquals("integer", mapper.typeName()); FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index e06d9889ec905..daa9fda7a5a3b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -74,14 +74,7 @@ public void testValidStarTree() throws IOException { for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; assertEquals(2, starTreeFieldType.getDimensions().size()); - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -92,7 +85,7 @@ public void testValidStarTree() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -137,14 +130,7 @@ public void testMetricsWithJustSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -154,7 +140,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -165,14 +151,7 @@ public void testMetricsWithCountAndSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -187,7 +166,7 @@ public void testMetricsWithCountAndSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("@timestamp", "status")), + new HashSet<>(Arrays.asList("node", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -198,14 +177,7 @@ public void testValidStarTreeDefaults() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); - assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); - DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY - ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(3, starTreeFieldType.getMetrics().size()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); @@ -278,7 +250,7 @@ public void testMissingDims() { MapperParsingException.class, () -> createMapperService(getMinMapping(false, false, true, false)) ); - assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [@timestamp]")); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [node]")); } public void testMissingMetrics() { @@ -314,7 +286,7 @@ public void testInvalidDimType() { () -> createMapperService(getInvalidMapping(false, false, true, false)) ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [@timestamp] as part of star tree field [startree]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [node] as part of star tree field [startree]", ex.getMessage() ); } @@ -511,7 +483,7 @@ public void testValidations() throws IOException { ) ); assertEquals( - "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field", + "Aggregations not supported for the dimension field [node] with field type [integer] as part of star tree field", ex.getMessage() ); @@ -534,17 +506,13 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -562,8 +530,8 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -585,7 +553,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("config") .startArray("ordered_dimensions") .startObject() - .field("name", "timestamp") + .field("name", "node") .endObject() .startObject() .field("name", "numeric_dv") @@ -606,8 +574,8 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .endObject() .endObject() .startObject("properties") - .startObject("timestamp") - .field("type", "date") + .startObject("node") + .field("type", "integer") .endObject() .startObject("numeric_dv") .field("type", "integer") @@ -634,17 +602,13 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -662,8 +626,8 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -684,17 +648,13 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("@timestamp"); + b.value("node"); b.value("status"); } b.endArray(); b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); - b.startArray("calendar_intervals"); - b.value("day"); - b.value("month"); - b.endArray(); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", dim); @@ -713,8 +673,8 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -740,7 +700,7 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric if (!isEmptyDims) { b.startArray("ordered_dimensions"); b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); b.startObject(); b.field("name", "status"); @@ -762,8 +722,8 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric b.endObject(); b.startObject("properties"); if (!missingDim) { - b.startObject("@timestamp"); - b.field("type", "date"); + b.startObject("node"); + b.field("type", "integer"); b.endObject(); } b.startObject("status"); @@ -876,7 +836,7 @@ private XContentBuilder getInvalidMapping( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); } b.startObject(); @@ -901,9 +861,9 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); + b.startObject("node"); if (!invalidDimType) { - b.field("type", "date"); + b.field("type", "integer"); } else { b.field("type", "keyword"); } @@ -946,7 +906,7 @@ private XContentBuilder getInvalidMappingWithDv( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "node"); b.endObject(); } b.startObject(); @@ -965,12 +925,12 @@ private XContentBuilder getInvalidMappingWithDv( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("@timestamp"); + b.startObject("node"); if (!invalidDimType) { - b.field("type", "date"); + b.field("type", "integer"); b.field("doc_values", "true"); } else { - b.field("type", "date"); + b.field("type", "integer"); b.field("doc_values", "false"); } b.endObject(); From d28fbc3bc2a4fbf0a70558e775e3f7a9e15d2617 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:11:12 +0530 Subject: [PATCH 044/357] =?UTF-8?q?Optimizing=20get=20indices=20to=20segme?= =?UTF-8?q?nts=20map=20calculations=20for=20IndicesSegmen=E2=80=A6=20(#147?= =?UTF-8?q?17)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Optimizing get indices to segments map calculations for IndicesSegmentResponse class Signed-off-by: Harsh Garg --- .../segments/IndicesSegmentResponse.java | 30 ++++++----- .../segments/IndicesSegmentResponseTests.java | 53 +++++++++++++++++++ 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java index 648f58dada4f9..b69d4b6f624b8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -45,13 +45,12 @@ import org.opensearch.index.engine.Segment; import java.io.IOException; -import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; /** * Transport response for retrieving indices segment information @@ -86,21 +85,24 @@ public Map getIndices() { return indicesSegments; } Map indicesSegments = new HashMap<>(); - - Set indices = new HashSet<>(); - for (ShardSegments shard : shards) { - indices.add(shard.getShardRouting().getIndexName()); + if (shards.length == 0) { + this.indicesSegments = indicesSegments; + return indicesSegments; } - for (String indexName : indices) { - List shards = new ArrayList<>(); - for (ShardSegments shard : this.shards) { - if (shard.getShardRouting().getIndexName().equals(indexName)) { - shards.add(shard); - } + Arrays.sort(shards, Comparator.comparing(shardSegment -> shardSegment.getShardRouting().getIndexName())); + int startIndexPos = 0; + String startIndexName = shards[startIndexPos].getShardRouting().getIndexName(); + for (int i = 0; i < shards.length; i++) { + if (!shards[i].getShardRouting().getIndexName().equals(startIndexName)) { + indicesSegments.put(startIndexName, new IndexSegments(startIndexName, Arrays.copyOfRange(shards, startIndexPos, i))); + startIndexPos = i; + startIndexName = shards[startIndexPos].getShardRouting().getIndexName(); } - indicesSegments.put(indexName, new IndexSegments(indexName, shards.toArray(new ShardSegments[0]))); } + // Add the last shardSegment from shards list which would have got missed in the loop above + indicesSegments.put(startIndexName, new IndexSegments(startIndexName, Arrays.copyOfRange(shards, startIndexPos, shards.length))); + this.indicesSegments = indicesSegments; return indicesSegments; } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponseTests.java index b90be87e3a600..c01335eb38afc 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/segments/IndicesSegmentResponseTests.java @@ -42,7 +42,10 @@ import org.opensearch.index.engine.Segment; import org.opensearch.test.OpenSearchTestCase; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; @@ -68,4 +71,54 @@ public void testToXContentSerialiationWithSortedFields() throws Exception { response.toXContent(builder, ToXContent.EMPTY_PARAMS); } } + + public void testGetIndices() { + final int totalIndices = 5; + final int shardsPerIndex = 3; + final int segmentsPerShard = 2; + // Preparing a ShardSegments list, which will have (totalIndices * shardsPerIndex) shardSegments. + // Indices will be named -> foo1, foo2, ..., foo{totalIndices} + List shardSegmentsList = new ArrayList<>(); + for (int indexName = 0; indexName < totalIndices; indexName++) { + for (int shardId = 0; shardId < shardsPerIndex; shardId++) { + ShardRouting shardRouting = TestShardRouting.newShardRouting( + "foo" + indexName, + shardId, + "node_id", + true, + ShardRoutingState.STARTED + ); + List segmentList = new ArrayList<>(); + for (int segmentNum = 0; segmentNum < segmentsPerShard; segmentNum++) { + segmentList.add(new Segment("foo" + indexName + shardId + segmentNum)); + } + shardSegmentsList.add(new ShardSegments(shardRouting, segmentList)); + } + } + Collections.shuffle(shardSegmentsList, random()); + + // Prepare the IndicesSegmentResponse object and get the indicesSegments map + IndicesSegmentResponse response = new IndicesSegmentResponse( + shardSegmentsList.toArray(new ShardSegments[0]), + totalIndices * shardsPerIndex, + totalIndices * shardsPerIndex, + 0, + Collections.emptyList() + ); + Map indicesSegments = response.getIndices(); + + assertEquals(totalIndices, indicesSegments.size()); + for (Map.Entry indexSegmentEntry : indicesSegments.entrySet()) { + assertEquals(shardsPerIndex, indexSegmentEntry.getValue().getShards().size()); + for (IndexShardSegments indexShardSegment : indexSegmentEntry.getValue().getShards().values()) { + for (ShardSegments shardSegment : indexShardSegment.getShards()) { + assertEquals(segmentsPerShard, shardSegment.getSegments().size()); + for (int i = 0; i < segmentsPerShard; i++) { + String segmentName = indexSegmentEntry.getKey() + shardSegment.getShardRouting().getId() + i; + assertEquals(segmentName, shardSegment.getSegments().get(i).getName()); + } + } + } + } + } } From 2793634f98819029e3bee327e1a47a89f8acc580 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:35:00 +0530 Subject: [PATCH 045/357] relaxing the join validation for nodes which have only store disabled but only publication enabled (#15471) Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 1 + .../coordination/JoinTaskExecutor.java | 45 ++++- .../metadata/RepositoriesMetadata.java | 18 ++ .../cluster/node/DiscoveryNode.java | 4 +- .../remotestore/RemoteStoreNodeAttribute.java | 13 ++ ...ransportClusterManagerNodeActionTests.java | 2 + .../coordination/JoinTaskExecutorTests.java | 157 +++++++++++++++--- .../MetadataCreateIndexServiceTests.java | 9 +- .../allocation/FailedShardsRoutingTests.java | 3 + ...eStoreMigrationAllocationDeciderTests.java | 2 + .../index/remote/RemoteStoreUtilsTests.java | 2 + .../index/shard/IndexShardTestUtils.java | 2 + 12 files changed, 235 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e255ad694206..f67e9a4c257f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) +- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index f77a7ffc8ce8e..838b5723b217b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -67,6 +67,7 @@ import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; @@ -458,7 +459,7 @@ public static void ensureNodesCompatibility( ); } - ensureRemoteStoreNodesCompatibility(joiningNode, currentNodes, metadata); + ensureRemoteRepositoryCompatibility(joiningNode, currentNodes, metadata); } /** @@ -491,6 +492,30 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) } } + public static void ensureRemoteRepositoryCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { + List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); + + boolean isClusterRemoteStoreEnabled = existingNodes.stream().anyMatch(DiscoveryNode::isRemoteStoreNode); + if (isClusterRemoteStoreEnabled || joiningNode.isRemoteStoreNode()) { + ensureRemoteStoreNodesCompatibility(joiningNode, currentNodes, metadata); + } else { + ensureRemoteClusterStateNodesCompatibility(joiningNode, currentNodes); + } + } + + private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes) { + List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); + + assert existingNodes.isEmpty() == false; + Optional remotePublicationNode = existingNodes.stream() + .filter(DiscoveryNode::isRemoteStatePublicationEnabled) + .findFirst(); + + if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { + ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); + } + } + /** * The method ensures homogeneity - * 1. The joining node has to be a remote store backed if it's joining a remote store backed cluster. Validates @@ -506,6 +531,7 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) * needs to be modified. */ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { + List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); assert existingNodes.isEmpty() == false; @@ -587,6 +613,23 @@ private static void ensureRemoteStoreNodesCompatibility( } } + private static void ensureRepositoryCompatibility(DiscoveryNode joiningNode, DiscoveryNode existingNode, List reposToValidate) { + + RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); + RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); + + if (existingRemoteStoreNodeAttribute.equalsForRepositories(joiningRemoteStoreNodeAttribute, reposToValidate) == false) { + throw new IllegalStateException( + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in " + + "comparison with existing node [" + + existingNode + + "]" + ); + } + } + public static Collection> addBuiltInJoinValidators( Collection> onJoinValidators ) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index 4b3dc7964a87b..59452e33191d7 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -184,6 +184,24 @@ public boolean equalsIgnoreGenerationsWithRepoSkip(@Nullable RepositoriesMetadat .filter(repo -> !reposToSkip.contains(repo.name())) .collect(Collectors.toList()); + return equalsRepository(currentRepositories, otherRepositories); + } + + public boolean equalsIgnoreGenerationsForRepo(@Nullable RepositoriesMetadata other, List reposToValidate) { + if (other == null) { + return false; + } + List currentRepositories = repositories.stream() + .filter(repo -> reposToValidate.contains(repo.name())) + .collect(Collectors.toList()); + List otherRepositories = other.repositories.stream() + .filter(repo -> reposToValidate.contains(repo.name())) + .collect(Collectors.toList()); + + return equalsRepository(currentRepositories, otherRepositories); + } + + public static boolean equalsRepository(List currentRepositories, List otherRepositories) { if (otherRepositories.size() != currentRepositories.size()) { return false; } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 3ed7633dce189..a6f0a457f7f9b 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -65,6 +65,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; /** * A discovery node represents a node that is part of the cluster. @@ -509,7 +510,8 @@ public boolean isSearchNode() { * @return true if the node contains remote store node attributes, false otherwise */ public boolean isRemoteStoreNode() { - return this.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)); + return this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) + && this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); } /** diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index efaefd530b34d..55971398634c5 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -55,6 +55,11 @@ public class RemoteStoreNodeAttribute { REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY ); + public static List REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES = List.of( + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + /** * Creates a new {@link RemoteStoreNodeAttribute} */ @@ -261,6 +266,14 @@ public boolean equalsWithRepoSkip(Object o, List reposToSkip) { return this.getRepositoriesMetadata().equalsIgnoreGenerationsWithRepoSkip(that.getRepositoriesMetadata(), reposToSkip); } + public boolean equalsForRepositories(Object otherNode, List repositoryToValidate) { + if (this == otherNode) return true; + if (otherNode == null || getClass() != otherNode.getClass()) return false; + + RemoteStoreNodeAttribute other = (RemoteStoreNodeAttribute) otherNode; + return this.getRepositoriesMetadata().equalsIgnoreGenerationsForRepo(other.repositoriesMetadata, repositoryToValidate); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index 37e884502b613..3c1c84653b384 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -85,6 +85,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdaterTests.createIndexMetadataWithRemoteStoreSettings; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; @@ -901,6 +902,7 @@ public void testDontAllowSwitchingCompatibilityModeForClusterWithMultipleVersion private Map getRemoteStoreNodeAttributes() { Map remoteStoreNodeAttributes = new HashMap<>(); + remoteStoreNodeAttributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-repo-1"); remoteStoreNodeAttributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1"); remoteStoreNodeAttributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1"); return remoteStoreNodeAttributes; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 9f463673aa6a6..d192a2556c36b 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -611,6 +611,100 @@ public void testJoinClusterWithRemoteStateNodeJoiningRemoteStateCluster() { JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } + public void testJoinRemotePublicationClusterWithNonRemoteNodes() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remotePublicationNodeAttributes(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(new HashMap<>()); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testJoinRemotePublicationCluster() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remotePublicationNodeAttributes(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remotePublicationNodeAttributes()); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + + public void testJoinRemotePubClusterWithRemoteStoreNodes() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remotePublicationNodeAttributes(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + Map newNodeAttributes = new HashMap<>(); + newNodeAttributes.putAll(remoteStateNodeAttributes(CLUSTER_STATE_REPO)); + newNodeAttributes.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + newNodeAttributes.putAll(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + + DiscoveryNode joiningNode = newDiscoveryNode(newNodeAttributes); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) + ); + assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster")); + } + + public void testPreventJoinRemotePublicationClusterWithIncompatibleAttributes() { + Map existingNodeAttributes = remotePublicationNodeAttributes(); + Map remoteStoreNodeAttributes = remotePublicationNodeAttributes(); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + existingNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), null); + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) + ); + assertTrue( + e.getMessage().equals("joining node [" + joiningNode + "] doesn't have the node attribute [" + nodeAttribute.getKey() + "]") + || e.getMessage() + .equals( + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in comparison with existing node [" + + currentState.getNodes().getNodes().values().stream().findFirst().get() + + "]" + ) + ); + + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); + } + } + public void testPreventJoinClusterWithRemoteStateNodeJoiningRemoteStoreCluster() { Map existingNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); final DiscoveryNode existingNode = new DiscoveryNode( @@ -628,16 +722,7 @@ public void testPreventJoinClusterWithRemoteStateNodeJoiningRemoteStoreCluster() IllegalStateException.class, () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); - assertTrue( - e.getMessage() - .equals( - "a remote store node [" - + joiningNode - + "] is trying to join a remote store cluster with incompatible node attributes in comparison with existing node [" - + currentState.getNodes().getNodes().values().stream().findFirst().get() - + "]" - ) - ); + assertTrue(e.getMessage().equals("a non remote store node [" + joiningNode + "] is trying to join a remote store cluster")); } public void testPreventJoinClusterWithRemoteStoreNodeJoiningRemoteStateCluster() { @@ -657,16 +742,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeJoiningRemoteStateCluster() IllegalStateException.class, () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); - assertTrue( - e.getMessage() - .equals( - "a remote store node [" - + joiningNode - + "] is trying to join a remote store cluster with incompatible node attributes in comparison with existing node [" - + currentState.getNodes().getNodes().values().stream().findFirst().get() - + "]" - ) - ); + assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster")); } public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { @@ -1077,6 +1153,39 @@ public void testRemoteRoutingTableNodeJoinNodeWithRemoteAndRoutingRepoDifference JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } + public void testJoinRemoteStoreClusterWithRemotePublicationNodeInMixedMode() { + final DiscoveryNode remoteStoreNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final DiscoveryNode nonRemoteStoreNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + new HashMap<>(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(remoteStoreNode).add(nonRemoteStoreNode).localNodeId(remoteStoreNode.getId()).build()) + .metadata(metadata) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remotePublicationNodeAttributes()); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); + } + private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories) throws Exception { @@ -1115,6 +1224,7 @@ private DiscoveryNode newDiscoveryNode(Map attributes) { } private static final String SEGMENT_REPO = "segment-repo"; + private static final String TRANSLOG_REPO = "translog-repo"; private static final String CLUSTER_STATE_REPO = "cluster-state-repo"; private static final String COMMON_REPO = "remote-repo"; @@ -1161,6 +1271,13 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St }; } + private Map remotePublicationNodeAttributes() { + Map existingNodeAttributes = new HashMap<>(); + existingNodeAttributes.putAll(remoteStateNodeAttributes(CLUSTER_STATE_REPO)); + existingNodeAttributes.putAll(remoteRoutingTableAttributes(ROUTING_TABLE_REPO)); + return existingNodeAttributes; + } + private Map remoteStateNodeAttributes(String clusterStateRepo) { String clusterStateRepositoryTypeAttributeKey = String.format( Locale.getDefault(), diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 410c8a3d41940..d376d0db8d6f4 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -171,6 +171,7 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.opensearch.node.Node.NODE_ATTRIBUTES; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo; @@ -1657,7 +1658,12 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() null ); - Map missingTranslogAttribute = Map.of(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1"); + Map missingTranslogAttribute = Map.of( + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + "cluster-state-repo-1", + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + "my-segment-repo-1" + ); DiscoveryNodes finalDiscoveryNodes = DiscoveryNodes.builder() .add(nonRemoteClusterManagerNode) @@ -2627,6 +2633,7 @@ private void verifyRemoteStoreIndexSettings( private DiscoveryNode getRemoteNode() { Map attributes = new HashMap<>(); + attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-rep-1"); attributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1"); attributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1"); return new DiscoveryNode( diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java index 5e3b74ee138ab..f8e1c609e6ee8 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java @@ -68,6 +68,7 @@ import static org.opensearch.cluster.routing.ShardRoutingState.STARTED; import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; @@ -853,6 +854,8 @@ public void testPreferReplicaOnRemoteNodeForPrimaryPromotion() { // add a remote node and start primary shard Map remoteStoreNodeAttributes = Map.of( + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + "REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY", REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_VALUE", REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java index 5b29922f2400c..e6e81c94e7f32 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java @@ -68,6 +68,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.NONE; import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE; import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; @@ -617,6 +618,7 @@ private DiscoveryNode getRemoteNode() { REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_VALUE" ); + attributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_VALUE"); return new DiscoveryNode( UUIDs.base64UUID(), buildNewFakeTransportAddress(), diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index a6db37285fe6f..2a34dd3580948 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -67,6 +67,7 @@ import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR; import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; @@ -540,6 +541,7 @@ private RoutingTable createRoutingTableAllShardsStarted( private Map getRemoteStoreNodeAttributes() { Map remoteStoreNodeAttributes = new HashMap<>(); + remoteStoreNodeAttributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-cluster-repo-1"); remoteStoreNodeAttributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1"); remoteStoreNodeAttributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1"); return remoteStoreNodeAttributes; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java index abf8f2a4da6c1..50a1751546293 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java @@ -21,6 +21,7 @@ import java.util.Map; public class IndexShardTestUtils { + public static final String MOCK_STATE_REPO_NAME = "state-test-repo"; public static final String MOCK_SEGMENT_REPO_NAME = "segment-test-repo"; public static final String MOCK_TLOG_REPO_NAME = "tlog-test-repo"; @@ -37,6 +38,7 @@ public static DiscoveryNode getFakeDiscoNode(String id) { public static DiscoveryNode getFakeRemoteEnabledNode(String id) { Map remoteNodeAttributes = new HashMap(); + remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_STATE_REPO_NAME); remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_SEGMENT_REPO_NAME); remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_TLOG_REPO_NAME); return new DiscoveryNode( From d2bc9fc3daaaa33273bace58c4a94d2ae3e7be5c Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Wed, 4 Sep 2024 17:18:09 +0530 Subject: [PATCH 046/357] Set Version to 2.17 for hashed prefix snapshots (#15656) Signed-off-by: Ashish Singh --- server/src/main/java/org/opensearch/repositories/IndexId.java | 4 ++-- .../main/java/org/opensearch/repositories/RepositoryData.java | 4 ++-- .../main/java/org/opensearch/snapshots/SnapshotsService.java | 2 +- .../test/java/org/opensearch/repositories/IndexIdTests.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 238dffbb46bde..5a9a757e31cb1 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -78,7 +78,7 @@ public IndexId(String name, String id, int shardPathType) { public IndexId(final StreamInput in) throws IOException { this.name = in.readString(); this.id = in.readString(); - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { this.shardPathType = in.readVInt(); } else { this.shardPathType = DEFAULT_SHARD_PATH_TYPE; @@ -145,7 +145,7 @@ private int computeHashCode() { public void writeTo(final StreamOutput out) throws IOException { out.writeString(name); out.writeString(id); - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeVInt(shardPathType); } } diff --git a/server/src/main/java/org/opensearch/repositories/RepositoryData.java b/server/src/main/java/org/opensearch/repositories/RepositoryData.java index 7921a9589b443..f52f1fa76f6bf 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoryData.java @@ -551,7 +551,7 @@ public List resolveNewIndices(List indicesToResolve, Map indexIds = repositoryData.resolveNewIndices( diff --git a/server/src/test/java/org/opensearch/repositories/IndexIdTests.java b/server/src/test/java/org/opensearch/repositories/IndexIdTests.java index 3b719d287aa9b..bcd75f9a47ad8 100644 --- a/server/src/test/java/org/opensearch/repositories/IndexIdTests.java +++ b/server/src/test/java/org/opensearch/repositories/IndexIdTests.java @@ -86,7 +86,7 @@ public void testEqualsAndHashCode() { public void testSerialization() throws IOException { IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID(), randomIntBetween(0, 2)); BytesStreamOutput out = new BytesStreamOutput(); - out.setVersion(Version.CURRENT); + out.setVersion(Version.V_2_17_0); indexId.writeTo(out); assertEquals(indexId, new IndexId(out.bytes().streamInput())); } From ad1df9e7b765d2e777cf77dec129a6cd8f11af8a Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Wed, 4 Sep 2024 20:57:09 +0800 Subject: [PATCH 047/357] fix wildcard term & terms query & add YAML REST test (#15607) * fix wildcard term & terms query & add YAML REST test Signed-off-by: gesong.samuel * add CHANGELOG Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Co-authored-by: gesong.samuel --- CHANGELOG.md | 1 + .../test/search/270_wildcard_fieldtype_queries.yml | 10 ++++++++++ .../opensearch/index/mapper/WildcardFieldMapper.java | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f67e9a4c257f7..71868de93812e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) - Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) +- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index 05b6b2e5ed712..c449a06cf633d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -227,3 +227,13 @@ setup: my_field: value: ".*" - match: { hits.total.value: 5 } +--- +"terms query on wildcard field matches": + - do: + search: + index: test + body: + query: + terms: { my_field: ["AbCd"] } + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "5" } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 4998a822917b4..cd95e320209ee 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -635,7 +635,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower @Override public Query termQueryCaseInsensitive(Object value, QueryShardContext context) { - return wildcardQuery(value.toString(), MultiTermQuery.CONSTANT_SCORE_REWRITE, true, context); + return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, true, context); } @Override @@ -649,7 +649,7 @@ public Query termsQuery(List values, QueryShardContext context) { Set expectedValues = new HashSet<>(); StringBuilder pattern = new StringBuilder(); for (Object value : values) { - String stringVal = value.toString(); + String stringVal = BytesRefs.toString(value); builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD); expectedValues.add(stringVal); if (pattern.length() > 0) { From 4f57d6aab21594a5ac20d26abbbba959aba1ab75 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 4 Sep 2024 05:58:28 -0700 Subject: [PATCH 048/357] Fix approximate pointrangequery tests (#15590) Signed-off-by: Harsha Vamsi Kalluri --- .../ApproximatePointRangeQuery.java | 15 ++++++++++----- .../ApproximatePointRangeQueryTests.java | 19 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java index cee8bc43d7ffd..8076da6ab970b 100644 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java @@ -146,10 +146,11 @@ public void grow(int count) { public void visit(int docID) { // it is possible that size < 1024 and docCount < size but we will continue to count through all the 1024 docs // and collect less, but it won't hurt performance - if (docCount[0] < size) { - adder.add(docID); - docCount[0]++; + if (docCount[0] >= size) { + return; } + adder.add(docID); + docCount[0]++; } @Override @@ -231,7 +232,7 @@ private void intersectRight(PointValues.PointTree pointTree, PointValues.Interse public void intersectLeft(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) throws IOException { PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); - if (docCount[0] > size) { + if (docCount[0] >= size) { return; } switch (r) { @@ -279,7 +280,7 @@ public void intersectLeft(PointValues.IntersectVisitor visitor, PointValues.Poin public void intersectRight(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) throws IOException { PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); - if (docCount[0] > size) { + if (docCount[0] >= size) { return; } switch (r) { @@ -435,6 +436,10 @@ public boolean canApproximate(SearchContext context) { if (!(context.query() instanceof ApproximateIndexOrDocValuesQuery)) { return false; } + // size 0 could be set for caching + if (context.from() + context.size() == 0) { + this.setSize(10_000); + } this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo())); if (context.request() != null && context.request().source() != null) { FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(context.request().source()); diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index 7fb75e43da6ec..1eaea6a9e9d47 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -127,9 +127,9 @@ public void testApproximateRangeWithSizeUnderDefault() throws IOException { } doc.add(new LongPoint("point", scratch)); iw.addDocument(doc); - if (i % 15 == 0) iw.flush(); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -168,6 +168,7 @@ public void testApproximateRangeWithSizeOverDefault() throws IOException { iw.addDocument(doc); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -185,7 +186,7 @@ protected String toString(int dimension, byte[] value) { }; IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(approximateQuery, 11000); - assertEquals(topDocs.totalHits, new TotalHits(11001, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); + assertEquals(topDocs.totalHits, new TotalHits(11000, TotalHits.Relation.EQUAL_TO)); } catch (IOException e) { throw new RuntimeException(e); } @@ -213,6 +214,7 @@ public void testApproximateRangeShortCircuit() throws IOException { if (i % 10 == 0) iw.flush(); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -258,6 +260,7 @@ public void testApproximateRangeShortCircuitAscSort() throws IOException { iw.addDocument(doc); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -284,12 +287,12 @@ protected String toString(int dimension, byte[] value) { assertNotEquals(topDocs.totalHits, topDocs1.totalHits); assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); assertEquals(topDocs1.totalHits, new TotalHits(21, TotalHits.Relation.EQUAL_TO)); - assertEquals(topDocs.scoreDocs[0].doc, 0); - assertEquals(topDocs.scoreDocs[1].doc, 1); - assertEquals(topDocs.scoreDocs[2].doc, 2); - assertEquals(topDocs.scoreDocs[3].doc, 3); - assertEquals(topDocs.scoreDocs[4].doc, 4); - assertEquals(topDocs.scoreDocs[5].doc, 5); + assertEquals(topDocs.scoreDocs[0].doc, topDocs1.scoreDocs[0].doc); + assertEquals(topDocs.scoreDocs[1].doc, topDocs1.scoreDocs[1].doc); + assertEquals(topDocs.scoreDocs[2].doc, topDocs1.scoreDocs[2].doc); + assertEquals(topDocs.scoreDocs[3].doc, topDocs1.scoreDocs[3].doc); + assertEquals(topDocs.scoreDocs[4].doc, topDocs1.scoreDocs[4].doc); + assertEquals(topDocs.scoreDocs[5].doc, topDocs1.scoreDocs[5].doc); } catch (IOException e) { throw new RuntimeException(e); From 71a771e938f5faf5f6d2d00e0ba27e99e170ed88 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 4 Sep 2024 22:57:14 +0530 Subject: [PATCH 049/357] [Star tree][Bug] Fix for derived metrics (#15640) * Fix for derived metrics Signed-off-by: Bharathwaj G * fixes for byte Signed-off-by: Bharathwaj G --------- Signed-off-by: Bharathwaj G --- .../Composite99DocValuesReader.java | 2 +- .../index/compositeindex/datacube/Metric.java | 16 +++++ .../startree/builder/BaseStarTreeBuilder.java | 7 +- .../builder/OffHeapStarTreeBuilder.java | 2 +- .../builder/OnHeapStarTreeBuilder.java | 2 +- .../fileformats/meta/StarTreeMetadata.java | 29 ++++++-- .../startree/index/StarTreeValues.java | 2 +- .../index/mapper/NumberFieldMapper.java | 4 +- .../index/mapper/StarTreeMapper.java | 6 +- .../StarTreeDocValuesFormatTests.java | 69 ++++++++++++++++--- .../datacube/startree/StarTreeTestUtils.java | 40 +++++++++-- .../builder/AbstractStarTreeBuilderTests.java | 28 +++++--- 12 files changed, 163 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java index ca55e8e4573ab..7901336151c8e 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java @@ -164,7 +164,7 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r // adding metric fields for (Metric metric : starTreeMetadata.getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { fields.add( fullyQualifiedFieldNameForStarTreeMetricsDocValues( compositeFieldName, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java index 9accb0201170a..be16f1e9886cd 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java @@ -13,6 +13,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -23,10 +24,18 @@ public class Metric implements ToXContent { private final String field; private final List metrics; + private final List baseMetrics; public Metric(String field, List metrics) { this.field = field; this.metrics = metrics; + this.baseMetrics = new ArrayList<>(); + for (MetricStat metricStat : metrics) { + if (metricStat.isDerivedMetric()) { + continue; + } + baseMetrics.add(metricStat); + } } public String getField() { @@ -37,6 +46,13 @@ public List getMetrics() { return metrics; } + /** + * Returns only the base metrics + */ + public List getBaseMetrics() { + return baseMetrics; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index ba4cb792c00df..a1d638616f2aa 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -150,10 +150,7 @@ public List generateMetricAggregatorInfos(MapperService ma metricAggregatorInfos.add(metricAggregatorInfo); continue; } - for (MetricStat metricStat : metric.getMetrics()) { - if (metricStat.isDerivedMetric()) { - continue; - } + for (MetricStat metricStat : metric.getBaseMetrics()) { FieldValueConverter fieldValueConverter; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof FieldMapper && ((FieldMapper) fieldMapper).fieldType() instanceof FieldValueConverter) { @@ -185,7 +182,7 @@ public List getMetricReaders(SegmentWriteState stat List metricReaders = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { SequentialDocValuesIterator metricReader; FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); if (metricStat.equals(MetricStat.DOC_COUNT)) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 62fda3e56d289..1613b7c5a3ac0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -158,7 +158,7 @@ Iterator mergeStarTrees(List starTreeValuesSub List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeValues.getStarTreeField().getName(), metric.getField(), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index ec9475caf7d6d..1a5c906ad413b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -144,7 +144,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeValues.getStarTreeField().getName(), metric.getField(), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index e7bb32282ece7..7352c215ee390 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -220,16 +220,37 @@ private int readMetricsCount() throws IOException { private List readMetricEntries() throws IOException { int metricCount = readMetricsCount(); - Map starTreeMetricMap = new LinkedHashMap<>(); + Map> starTreeMetricStatMap = new LinkedHashMap<>(); for (int i = 0; i < metricCount; i++) { String metricName = meta.readString(); int metricStatOrdinal = meta.readVInt(); MetricStat metricStat = MetricStat.fromMetricOrdinal(metricStatOrdinal); - Metric metric = starTreeMetricMap.computeIfAbsent(metricName, field -> new Metric(field, new ArrayList<>())); - metric.getMetrics().add(metricStat); + List metricStats = starTreeMetricStatMap.computeIfAbsent(metricName, field -> new ArrayList<>()); + metricStats.add(metricStat); } + List starTreeMetricMap = new ArrayList<>(); + for (Map.Entry> metricStatsEntry : starTreeMetricStatMap.entrySet()) { + addEligibleDerivedMetrics(metricStatsEntry.getValue()); + starTreeMetricMap.add(new Metric(metricStatsEntry.getKey(), metricStatsEntry.getValue())); - return new ArrayList<>(starTreeMetricMap.values()); + } + return starTreeMetricMap; + } + + /** + * Add derived metrics if all associated base metrics are present + */ + private void addEligibleDerivedMetrics(List metricStatsList) { + Set metricStatsSet = new HashSet<>(metricStatsList); + for (MetricStat metric : MetricStat.values()) { + if (metric.isDerivedMetric() && !metricStatsSet.contains(metric)) { + List sourceMetrics = metric.getBaseMetrics(); + if (metricStatsSet.containsAll(sourceMetrics)) { + metricStatsList.add(metric); + metricStatsSet.add(metric); + } + } + } } private int readSegmentAggregatedDocCount() throws IOException { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 7d5f5ba02b9f8..a34bbbe9ee738 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -171,7 +171,7 @@ public StarTreeValues( // get doc id set iterators for metrics for (Metric metric : starTreeMetadata.getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeField.getName(), metric.getField(), diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index eb0694edc70ba..43e975f95757b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -653,9 +653,7 @@ public byte[] encodePoint(Number value) { @Override public double toDoubleValue(long value) { - byte[] bytes = new byte[8]; - NumericUtils.longToSortableBytes(value, bytes, 0); - return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); + return objectToDouble(value); } @Override diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index 9e2e7ef3c3889..52dab17e0b0bb 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -287,11 +287,7 @@ private List buildMetrics(String fieldName, Map map, Map } int numBaseMetrics = 0; for (Metric metric : metrics) { - for (MetricStat metricStat : metric.getMetrics()) { - if (metricStat.isDerivedMetric() == false) { - numBaseMetrics++; - } - } + numBaseMetrics += metric.getBaseMetrics().size(); } if (numBaseMetrics > context.getSettings() .getAsInt( diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java index 0c6d21d28cc8a..1c267c67e60ed 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java @@ -119,23 +119,23 @@ public void testStarTreeDocValues() throws IOException { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); iw.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", 1)); + doc.add(new SortedNumericDocValuesField("field", -1)); iw.addDocument(doc); doc = new Document(); iw.forceMerge(1); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); iw.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 2)); doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", 2)); + doc.add(new SortedNumericDocValuesField("field", -2)); iw.addDocument(doc); iw.forceMerge(1); iw.close(); @@ -144,11 +144,39 @@ public void testStarTreeDocValues() throws IOException { TestUtil.checkReader(ir); assertEquals(1, ir.leaves().size()); + // Segment documents + /** + * sndv dv field + * [1, 1, -1] + * [1, 1, -1] + * [2, 2, -2] + * [2, 2, -2] + */ + // Star tree docuements + /** + * sndv dv | [ sum, value_count, min, max[field]] , [ sum, value_count, min, max[sndv]], doc_count + * [1, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [2, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + * [null, 1] | [-2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0] + * [null, 2] | [-4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0] + */ StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[4]; - expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 1L }, new Double[] { 2.0, 2.0, 2.0 }); - expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 2L, 2L }, new Double[] { 4.0, 2.0, 4.0 }); - expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 2.0, 2.0, 2.0 }); - expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, 2L }, new Double[] { 4.0, 2.0, 4.0 }); + expectedStarTreeDocuments[0] = new StarTreeDocument( + new Long[] { 1L, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[1] = new StarTreeDocument( + new Long[] { 2L, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); + expectedStarTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, 1L }, + new Double[] { -2.0, 2.0, -1.0, -1.0, 2.0, 2.0, 1.0, 1.0, 2.0 } + ); + expectedStarTreeDocuments[3] = new StarTreeDocument( + new Long[] { null, 2L }, + new Double[] { -4.0, 2.0, -2.0, -2.0, 4.0, 2.0, 2.0, 2.0, 2.0 } + ); for (LeafReaderContext context : ir.leaves()) { SegmentReader reader = Lucene.segmentReader(context.reader()); @@ -159,7 +187,17 @@ public void testStarTreeDocValues() throws IOException { StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( List.of(starTreeValues), - List.of(NumberFieldMapper.NumberType.DOUBLE, NumberFieldMapper.NumberType.LONG, NumberFieldMapper.NumberType.LONG), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), reader.maxDoc() ); assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); @@ -190,6 +228,19 @@ private XContentBuilder getExpandedMapping() throws IOException { b.startArray("stats"); b.value("sum"); b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.startObject(); + b.field("name", "sndv"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); b.endArray(); b.endObject(); b.endArray(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index f3e111bf6caa6..b7395b993f67b 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -61,6 +61,9 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { + if (metricStat.isDerivedMetric()) { + continue; + } String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeValues.getStarTreeField().getName(), metric.getField(), @@ -125,18 +128,18 @@ public static void assertStarTreeDocuments(StarTreeDocument[] starTreeDocuments, assertNotNull(resultStarTreeDocument.dimensions); assertNotNull(resultStarTreeDocument.metrics); - assertEquals(resultStarTreeDocument.dimensions.length, expectedStarTreeDocument.dimensions.length); - assertEquals(resultStarTreeDocument.metrics.length, expectedStarTreeDocument.metrics.length); + assertEquals(expectedStarTreeDocument.dimensions.length, resultStarTreeDocument.dimensions.length); + assertEquals(expectedStarTreeDocument.metrics.length, resultStarTreeDocument.metrics.length); for (int di = 0; di < resultStarTreeDocument.dimensions.length; di++) { - assertEquals(resultStarTreeDocument.dimensions[di], expectedStarTreeDocument.dimensions[di]); + assertEquals(expectedStarTreeDocument.dimensions[di], resultStarTreeDocument.dimensions[di]); } for (int mi = 0; mi < resultStarTreeDocument.metrics.length; mi++) { if (expectedStarTreeDocument.metrics[mi] instanceof Long) { - assertEquals(resultStarTreeDocument.metrics[mi], ((Long) expectedStarTreeDocument.metrics[mi]).doubleValue()); + assertEquals(((Long) expectedStarTreeDocument.metrics[mi]).doubleValue(), resultStarTreeDocument.metrics[mi]); } else { - assertEquals(resultStarTreeDocument.metrics[mi], expectedStarTreeDocument.metrics[mi]); + assertEquals(expectedStarTreeDocument.metrics[mi], resultStarTreeDocument.metrics[mi]); } } } @@ -267,9 +270,34 @@ public static void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetad Metric expectedMetric = expectedStarTreeMetadata.getMetrics().get(i); Metric resultMetric = resultStarTreeMetadata.getMetrics().get(i); assertEquals(expectedMetric.getField(), resultMetric.getField()); + List metricStats = new ArrayList<>(); + for (MetricStat metricStat : expectedMetric.getMetrics()) { + if (metricStat.isDerivedMetric()) { + continue; + } + metricStats.add(metricStat); + } + Metric expectedMetricWithoutDerivedMetrics = new Metric(expectedMetric.getField(), metricStats); + metricStats = new ArrayList<>(); + for (MetricStat metricStat : resultMetric.getMetrics()) { + if (metricStat.isDerivedMetric()) { + continue; + } + metricStats.add(metricStat); + } + Metric resultantMetricWithoutDerivedMetrics = new Metric(resultMetric.getField(), metricStats); + + // assert base metrics are in order in metadata + for (int j = 0; j < expectedMetricWithoutDerivedMetrics.getMetrics().size(); j++) { + assertEquals( + expectedMetricWithoutDerivedMetrics.getMetrics().get(j), + resultantMetricWithoutDerivedMetrics.getMetrics().get(j) + ); + } + // assert all metrics ( including derived metrics are present ) for (int j = 0; j < expectedMetric.getMetrics().size(); j++) { - assertEquals(expectedMetric.getMetrics().get(j), resultMetric.getMetrics().get(j)); + assertTrue(resultMetric.getMetrics().contains(expectedMetric.getMetrics().get(j))); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index ad54bda9a916e..65adc43ea8bea 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -205,9 +205,7 @@ private SegmentReadState getReadState(int numDocs, List dimensionFields, int numMetrics = 0; for (Metric metric : metrics) { - for (MetricStat metricStat : metric.getMetrics()) { - numMetrics++; - } + numMetrics += metric.getBaseMetrics().size(); } FieldInfo[] fields = new FieldInfo[dimensionFields.size() + numMetrics]; @@ -237,7 +235,7 @@ private SegmentReadState getReadState(int numDocs, List dimensionFields, } for (Metric metric : metrics) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { fields[i] = new FieldInfo( fullyQualifiedFieldNameForStarTreeMetricsDocValues( compositeField.getName(), @@ -2047,7 +2045,7 @@ public void testFlushFlow() throws IOException { VERSION_CURRENT, builder.numStarTreeNodes, List.of("field1", "field3"), - List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT))), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), 6, builder.numStarTreeDocs, 1000, @@ -2138,7 +2136,7 @@ public void testFlushFlowDimsReverse() throws IOException { VERSION_CURRENT, builder.numStarTreeNodes, List.of("field1", "field3"), - List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT))), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), 6, builder.numStarTreeDocs, 1000, @@ -2270,8 +2268,9 @@ private StarTreeField getStarTreeFieldWithMultipleMetrics() { Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); List dims = List.of(d1, d2); - List metrics = List.of(m1, m2); + List metrics = List.of(m1, m2, m3); StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); return new StarTreeField("sf", dims, metrics, c); } @@ -4069,12 +4068,19 @@ public void testMergeFlow() throws IOException { metricsWithField.add(i); } + List metricsListValueCount = new ArrayList<>(1000); + List metricsWithFieldValueCount = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsListValueCount.add((long) i); + metricsWithFieldValueCount.add(i); + } + Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); Dimension d3 = new NumericDimension("field5"); Dimension d4 = new NumericDimension("field8"); // Dimension d5 = new NumericDimension("field5"); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); List dims = List.of(d1, d2, d3, d4); List metrics = List.of(m1, m2); @@ -4085,6 +4091,7 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); Map> dimDocIdSetIterators = Map.of( "field1", @@ -4100,6 +4107,8 @@ public void testMergeFlow() throws IOException { Map> metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", () -> m1sndv, + "sf_field2_value_count_metric", + () -> valucountsndv, "sf__doc_count_doc_count_metric", () -> m2sndv ); @@ -4118,6 +4127,7 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2d3sndv = getSortedNumericMock(dimList3, docsWithField3); SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); Map> f2dimDocIdSetIterators = Map.of( "field1", @@ -4133,6 +4143,8 @@ public void testMergeFlow() throws IOException { Map> f2metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", () -> f2m1sndv, + "sf_field2_value_count_metric", + () -> f2valucountsndv, "sf__doc_count_doc_count_metric", () -> f2m2sndv ); From 3681b52081fc6e2f9c158800ce5639405811d792 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Sep 2024 12:17:04 -0700 Subject: [PATCH 050/357] [RW Separation] Add routing preference to route requests only to search replicas. (#15563) * Add routing preference to route requests only to search replicas. This adds SEARCH_REPLICA routing preference and defaults to this preference for indices that have search replicas. Signed-off-by: Marc Handalian * add changelog entry Signed-off-by: Marc Handalian * PR feedback - extract a private method for replica filtering Signed-off-by: Marc Handalian * remove changelog entry Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../indices/settings/SearchOnlyReplicaIT.java | 37 ++++++++- .../routing/IndexShardRoutingTable.java | 28 ++++--- .../cluster/routing/OperationRouting.java | 12 +++ .../cluster/routing/Preference.java | 7 ++ .../routing/OperationRoutingTests.java | 76 +++++++++++++++++++ .../ClusterStateCreationUtils.java | 26 +++++++ 6 files changed, 176 insertions(+), 10 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java index 5fc8e30ed2c7a..6bd91df1de66f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java @@ -8,14 +8,17 @@ package org.opensearch.indices.settings; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -110,7 +113,6 @@ public void testFailoverWithSearchReplica_WithWriterReplicas() throws IOExceptio // add back a node internalCluster().startDataOnlyNode(); ensureGreen(TEST_INDEX); - } public void testFailoverWithSearchReplica_WithoutWriterReplicas() throws IOException { @@ -175,6 +177,39 @@ public void testSearchReplicaScaling() { assertActiveSearchShards(0); } + public void testSearchReplicaRoutingPreference() throws IOException { + int numSearchReplicas = 1; + int numWriterReplicas = 1; + internalCluster().startClusterManagerOnlyNode(); + String primaryNodeName = internalCluster().startDataOnlyNode(); + createIndex( + TEST_INDEX, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numWriterReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, numSearchReplicas) + .build() + ); + ensureYellow(TEST_INDEX); + client().prepareIndex(TEST_INDEX).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + // add 2 nodes for the replicas + internalCluster().startDataOnlyNodes(2); + ensureGreen(TEST_INDEX); + + assertActiveShardCounts(numSearchReplicas, numWriterReplicas); + + // set preference to search replica here - we default to this when there are + // search replicas but tests will randomize this value if unset + SearchResponse response = client().prepareSearch(TEST_INDEX) + .setPreference(Preference.SEARCH_REPLICA.type()) + .setQuery(QueryBuilders.matchAllQuery()) + .get(); + + String nodeId = response.getHits().getAt(0).getShard().getNodeId(); + IndexShardRoutingTable indexShardRoutingTable = getIndexShardRoutingTable(); + assertEquals(nodeId, indexShardRoutingTable.searchOnlyReplicas().get(0).currentNodeId()); + } + /** * Helper to assert counts of active shards for each type. */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index 4cc3300676986..f25cb14f65eca 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -647,15 +647,11 @@ public ShardIterator replicaActiveInitializingShardIt() { return new PlainShardIterator(shardId, Collections.emptyList()); } - LinkedList ordered = new LinkedList<>(); - for (ShardRouting replica : shuffler.shuffle(replicas)) { - if (replica.active()) { - ordered.addFirst(replica); - } else if (replica.initializing()) { - ordered.addLast(replica); - } - } - return new PlainShardIterator(shardId, ordered); + return filterAndOrderShards(replica -> true); + } + + public ShardIterator searchReplicaActiveInitializingShardIt() { + return filterAndOrderShards(ShardRouting::isSearchOnly); } /** @@ -686,6 +682,20 @@ public ShardIterator replicaFirstActiveInitializingShardsIt() { return new PlainShardIterator(shardId, ordered); } + private ShardIterator filterAndOrderShards(Predicate filter) { + LinkedList ordered = new LinkedList<>(); + for (ShardRouting replica : shuffler.shuffle(replicas)) { + if (filter.test(replica)) { + if (replica.active()) { + ordered.addFirst(replica); + } else if (replica.initializing()) { + ordered.addLast(replica); + } + } + } + return new PlainShardIterator(shardId, ordered); + } + /** * Returns an iterator on active and initializing shards residing on the provided nodeId. */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java index 6242247f34a93..fe9e00b250e70 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/OperationRouting.java @@ -121,6 +121,7 @@ public class OperationRouting { private volatile boolean isFailOpenEnabled; private volatile boolean isStrictWeightedShardRouting; private volatile boolean ignoreWeightedRouting; + private final boolean isReaderWriterSplitEnabled; public OperationRouting(Settings settings, ClusterSettings clusterSettings) { // whether to ignore awareness attributes when routing requests @@ -141,6 +142,7 @@ public OperationRouting(Settings settings, ClusterSettings clusterSettings) { clusterSettings.addSettingsUpdateConsumer(WEIGHTED_ROUTING_FAILOPEN_ENABLED, this::setFailOpenEnabled); clusterSettings.addSettingsUpdateConsumer(STRICT_WEIGHTED_SHARD_ROUTING_ENABLED, this::setStrictWeightedShardRouting); clusterSettings.addSettingsUpdateConsumer(IGNORE_WEIGHTED_SHARD_ROUTING, this::setIgnoreWeightedRouting); + this.isReaderWriterSplitEnabled = FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.get(settings); } void setUseAdaptiveReplicaSelection(boolean useAdaptiveReplicaSelection) { @@ -254,6 +256,14 @@ public GroupShardsIterator searchShards( preference = Preference.PRIMARY_FIRST.type(); } + if (isReaderWriterSplitEnabled) { + if (preference == null || preference.isEmpty()) { + if (indexMetadataForShard.getNumberOfSearchOnlyReplicas() > 0) { + preference = Preference.SEARCH_REPLICA.type(); + } + } + } + ShardIterator iterator = preferenceActiveShardIterator( shard, clusterState.nodes().getLocalNodeId(), @@ -366,6 +376,8 @@ private ShardIterator preferenceActiveShardIterator( return indexShard.primaryFirstActiveInitializingShardsIt(); case REPLICA_FIRST: return indexShard.replicaFirstActiveInitializingShardsIt(); + case SEARCH_REPLICA: + return indexShard.searchReplicaActiveInitializingShardIt(); case ONLY_LOCAL: return indexShard.onlyNodeActiveInitializingShardsIt(localNodeId); case ONLY_NODES: diff --git a/server/src/main/java/org/opensearch/cluster/routing/Preference.java b/server/src/main/java/org/opensearch/cluster/routing/Preference.java index a1ea01afa118f..093e3d5fd45f8 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/Preference.java +++ b/server/src/main/java/org/opensearch/cluster/routing/Preference.java @@ -73,6 +73,11 @@ public enum Preference { */ REPLICA_FIRST("_replica_first"), + /** + * Route to search replica shards + */ + SEARCH_REPLICA("_search_replica"), + /** * Route to the local shard only */ @@ -127,6 +132,8 @@ public static Preference parse(String preference) { return ONLY_LOCAL; case "_only_nodes": return ONLY_NODES; + case "_search_replica": + return SEARCH_REPLICA; default: throw new IllegalArgumentException("no Preference for [" + preferenceType + "]"); } diff --git a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java index ad8b48d56c417..aaeeb52ab5709 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/OperationRoutingTests.java @@ -1118,6 +1118,82 @@ public void testPartialIndexPrimaryDefault() throws Exception { } } + public void testSearchReplicaDefaultRouting() throws Exception { + final int numShards = 1; + final int numReplicas = 2; + final int numSearchReplicas = 2; + final String indexName = "test"; + final String[] indexNames = new String[] { indexName }; + + ClusterService clusterService = null; + ThreadPool threadPool = null; + + try { + OperationRouting opRouting = new OperationRouting( + Settings.builder().put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, "true").build(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + + ClusterState state = ClusterStateCreationUtils.stateWithAssignedPrimariesAndReplicas( + indexNames, + numShards, + numReplicas, + numSearchReplicas + ); + IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index(indexName).getShards().get(0); + ShardId shardId = indexShardRoutingTable.searchOnlyReplicas().get(0).shardId(); + + threadPool = new TestThreadPool("testSearchReplicaDefaultRouting"); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + + // add a search replica in initializing state: + DiscoveryNode node = new DiscoveryNode( + "node_initializing", + OpenSearchTestCase.buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), + Version.CURRENT + ); + + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(Settings.builder().put(state.metadata().index(indexName).getSettings()).build()) + .numberOfSearchReplicas(3) + .numberOfReplicas(2) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder(state.metadata()).put(indexMetadata, false).generateClusterUuidIfNeeded(); + IndexRoutingTable.Builder indexShardRoutingBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); + indexShardRoutingBuilder.addIndexShard(indexShardRoutingTable); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(shardId, node.getId(), null, false, true, ShardRoutingState.INITIALIZING, null) + ); + state = ClusterState.builder(state) + .routingTable(RoutingTable.builder().add(indexShardRoutingBuilder).build()) + .metadata(metadataBuilder.build()) + .build(); + + // Verify default preference is primary only + GroupShardsIterator groupIterator = opRouting.searchShards(state, indexNames, null, null); + assertThat("one group per shard", groupIterator.size(), equalTo(numShards)); + for (ShardIterator shardIterator : groupIterator) { + assertEquals("We should have 3 shards returned", shardIterator.size(), 3); + int i = 0; + for (ShardRouting shardRouting : shardIterator) { + assertTrue( + "Only search replicas should exist with preference SEARCH_REPLICA", + shardIterator.nextOrNull().isSearchOnly() + ); + if (i == shardIterator.size()) { + assertTrue("Initializing shard should appear last", shardRouting.initializing()); + assertFalse("Initializing shard should appear last", shardRouting.active()); + } + } + } + } finally { + IOUtils.close(clusterService); + terminate(threadPool); + } + } + private DiscoveryNode[] setupNodes() { // Sets up two data nodes in zone-a and one data node in zone-b List zones = Arrays.asList("a", "a", "b"); diff --git a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java index 8650500df8e95..0c4e871b1330c 100644 --- a/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/opensearch/action/support/replication/ClusterStateCreationUtils.java @@ -63,6 +63,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.test.OpenSearchTestCase.randomFrom; @@ -325,7 +326,18 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index, * Creates cluster state with several indexes, shards and replicas and all shards STARTED. */ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indices, int numberOfShards, int numberOfReplicas) { + return stateWithAssignedPrimariesAndReplicas(indices, numberOfShards, numberOfReplicas, 0); + } + /** + * Creates cluster state with several indexes, shards and replicas and all shards STARTED. + */ + public static ClusterState stateWithAssignedPrimariesAndReplicas( + String[] indices, + int numberOfShards, + int numberOfReplicas, + int numberOfSearchReplicas + ) { int numberOfDataNodes = numberOfReplicas + 1; DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); for (int i = 0; i < numberOfDataNodes + 1; i++) { @@ -347,6 +359,7 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice .put(SETTING_VERSION_CREATED, Version.CURRENT) .put(SETTING_NUMBER_OF_SHARDS, numberOfShards) .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, numberOfSearchReplicas) .put(SETTING_CREATION_DATE, System.currentTimeMillis()) ) .build(); @@ -363,6 +376,19 @@ public static ClusterState stateWithAssignedPrimariesAndReplicas(String[] indice TestShardRouting.newShardRouting(index, i, newNode(replica + 1).getId(), null, false, ShardRoutingState.STARTED) ); } + for (int replica = numberOfReplicas; replica < numberOfSearchReplicas + numberOfReplicas; replica++) { + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting( + new ShardId(index, IndexMetadata.INDEX_UUID_NA_VALUE, i), + newNode(replica + 1).getId(), + null, + false, + true, + ShardRoutingState.STARTED, + null + ) + ); + } indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build()); } routingTableBuilder.add(indexRoutingTableBuilder.build()); From 7b0846ecb1ec3707cf996b81e74559bff1442dd5 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 5 Sep 2024 00:55:14 +0530 Subject: [PATCH 051/357] Integrate translog cleanup with snapshot deletion and fix primary term deletion logic (#15657) --------- Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- ...pshotITV2.java => DeleteSnapshotV2IT.java} | 11 +- .../store/RemoteSegmentStoreDirectory.java | 2 +- .../RemoteSegmentStoreDirectoryFactory.java | 4 + .../RemoteFsTimestampAwareTranslog.java | 172 ++++++++++++++++-- .../transfer/TranslogTransferManager.java | 8 + .../transfer/TranslogTransferMetadata.java | 50 ++++- .../blobstore/BlobStoreRepository.java | 46 ++++- .../index/remote/RemoteStoreUtilsTests.java | 4 +- .../RemoteSegmentStoreDirectoryTests.java | 8 +- .../RemoteFsTimestampAwareTranslogTests.java | 160 ++++++++++++++-- .../TranslogTransferManagerTests.java | 2 +- 11 files changed, 422 insertions(+), 45 deletions(-) rename server/src/internalClusterTest/java/org/opensearch/snapshots/{DeleteSnapshotITV2.java => DeleteSnapshotV2IT.java} (96%) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java similarity index 96% rename from server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java rename to server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index 02b6ea47172c7..44d5c0a28cd9a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotITV2.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.lessThan; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class DeleteSnapshotITV2 extends AbstractSnapshotIntegTestCase { +public class DeleteSnapshotV2IT extends AbstractSnapshotIntegTestCase { private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; @@ -276,9 +276,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); Path shardPath = Path.of(String.valueOf(indexPath), "0"); Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog"); // Get total segments remote store directory file count for deleted index and shard 0 int segmentFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); + int translogFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(translogPath); RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); @@ -312,6 +314,13 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountAfterDeletingSnapshot1)); } catch (Exception e) {} }, 60, TimeUnit.SECONDS); + + assertBusy(() -> { + try { + assertThat(RemoteStoreBaseIntegTestCase.getFileCount(translogPath), lessThan(translogFilesCountBeforeDeletingSnapshot1)); + } catch (Exception e) {} + }, 60, TimeUnit.SECONDS); + } private Settings snapshotV2Settings(Path remoteStoreRepoPath) { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 53b43bbfb3bba..72bf07d4b03b2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -412,7 +412,7 @@ static long getGeneration(String[] filenameTokens) { public static long getTimestamp(String filename) { String[] filenameTokens = filename.split(SEPARATOR); - return RemoteStoreUtils.invertLong(filenameTokens[6]); + return RemoteStoreUtils.invertLong(filenameTokens[filenameTokens.length - 2]); } public static Tuple getNodeIdByPrimaryTermAndGen(String filename) { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index ea7e2506deec3..233665e65aed9 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -116,4 +116,8 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s } } + public Supplier getRepositoriesService() { + return this.repositoriesService; + } + } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 0b134b3bddbec..27d34ec0d05af 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -8,6 +8,7 @@ package org.opensearch.index.translog; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobMetadata; @@ -33,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; import java.util.function.LongSupplier; @@ -52,10 +54,13 @@ */ public class RemoteFsTimestampAwareTranslog extends RemoteFsTranslog { + private static Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslog.class); private final Logger logger; private final Map metadataFilePinnedTimestampMap; // For metadata files, with no min generation in the name, we cache generation data to avoid multiple reads. private final Map> oldFormatMetadataFileGenerationMap; + private final Map> oldFormatMetadataFilePrimaryTermMap; + private final AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); public RemoteFsTimestampAwareTranslog( TranslogConfig config, @@ -86,6 +91,7 @@ public RemoteFsTimestampAwareTranslog( logger = Loggers.getLogger(getClass(), shardId); this.metadataFilePinnedTimestampMap = new HashMap<>(); this.oldFormatMetadataFileGenerationMap = new HashMap<>(); + this.oldFormatMetadataFilePrimaryTermMap = new HashMap<>(); } @Override @@ -165,7 +171,11 @@ public void onResponse(List blobMetadata) { return; } - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted( + metadataFiles, + metadataFilePinnedTimestampMap, + logger + ); // If index is not deleted, make sure to keep latest metadata file if (indexDeleted == false) { @@ -209,7 +219,7 @@ public void onResponse(List blobMetadata) { oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); // Delete stale primary terms - deleteStaleRemotePrimaryTerms(metadataFiles); + deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted); } else { remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); } @@ -259,8 +269,16 @@ protected Set getGenerationsToBeDeleted( return generationsToBeDeleted; } - // Visible for testing protected List getMetadataFilesToBeDeleted(List metadataFiles) { + return getMetadataFilesToBeDeleted(metadataFiles, metadataFilePinnedTimestampMap, logger); + } + + // Visible for testing + protected static List getMetadataFilesToBeDeleted( + List metadataFiles, + Map metadataFilePinnedTimestampMap, + Logger logger + ) { Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); // Keep files since last successful run of scheduler @@ -351,27 +369,153 @@ protected Tuple getMinMaxTranslogGenerationFromMetadataFile( } } + private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + oldFormatMetadataFilePrimaryTermMap, + minPrimaryTermInRemote, + logger + ); + } + /** * This method must be called only after there are valid generations to delete in trimUnreferencedReaders as it ensures * implicitly that minimum primary term in latest translog metadata in remote store is the current primary term. *
* This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ - private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + protected static void deleteStaleRemotePrimaryTerms( + List metadataFiles, + TranslogTransferManager translogTransferManager, + Map> oldFormatMetadataFilePrimaryTermMap, + AtomicLong minPrimaryTermInRemoteAtomicLong, + Logger logger + ) { // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. - if (olderPrimaryCleaned.trySet(Boolean.TRUE)) { - if (metadataFiles.isEmpty()) { - logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); - return; + if (metadataFiles.isEmpty()) { + logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); + return; + } + Optional minPrimaryTermFromMetadataFiles = metadataFiles.stream().map(file -> { + try { + return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1(); + } catch (IOException e) { + return Long.MAX_VALUE; + } + }).min(Long::compareTo); + // First we delete all stale primary terms folders from remote store + Long minPrimaryTermInRemote = getMinPrimaryTermInRemote(minPrimaryTermInRemoteAtomicLong, translogTransferManager, logger); + if (minPrimaryTermFromMetadataFiles.get() > minPrimaryTermInRemote) { + translogTransferManager.deletePrimaryTermsAsync(minPrimaryTermFromMetadataFiles.get()); + minPrimaryTermInRemoteAtomicLong.set(minPrimaryTermFromMetadataFiles.get()); + } else { + logger.debug( + "Skipping primary term cleanup. minimumReferencedPrimaryTerm = {}, minPrimaryTermInRemote = {}", + minPrimaryTermFromMetadataFiles.get(), + minPrimaryTermInRemote + ); + } + } + + private static Long getMinPrimaryTermInRemote( + AtomicLong minPrimaryTermInRemote, + TranslogTransferManager translogTransferManager, + Logger logger + ) { + if (minPrimaryTermInRemote.get() == Long.MAX_VALUE) { + try { + Set primaryTermsInRemote = translogTransferManager.listPrimaryTermsInRemote(); + if (primaryTermsInRemote.isEmpty() == false) { + Optional minPrimaryTerm = primaryTermsInRemote.stream().min(Long::compareTo); + minPrimaryTerm.ifPresent(minPrimaryTermInRemote::set); + } + } catch (IOException e) { + logger.error("Exception while listing primary terms in remote translog", e); + } + } + return minPrimaryTermInRemote.get(); + } + + protected static Tuple getMinMaxPrimaryTermFromMetadataFile( + String metadataFile, + TranslogTransferManager translogTransferManager, + Map> oldFormatMetadataFilePrimaryTermMap + ) throws IOException { + Tuple minMaxPrimaryTermFromFileName = TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(metadataFile); + if (minMaxPrimaryTermFromFileName != null) { + return minMaxPrimaryTermFromFileName; + } else { + if (oldFormatMetadataFilePrimaryTermMap.containsKey(metadataFile)) { + return oldFormatMetadataFilePrimaryTermMap.get(metadataFile); + } else { + TranslogTransferMetadata metadata = translogTransferManager.readMetadata(metadataFile); + long maxPrimaryTem = TranslogTransferMetadata.getPrimaryTermFromFileName(metadataFile); + long minPrimaryTem = -1; + if (metadata.getGenerationToPrimaryTermMapper() != null + && metadata.getGenerationToPrimaryTermMapper().values().isEmpty() == false) { + Optional primaryTerm = metadata.getGenerationToPrimaryTermMapper() + .values() + .stream() + .map(s -> Long.parseLong(s)) + .min(Long::compareTo); + if (primaryTerm.isPresent()) { + minPrimaryTem = primaryTerm.get(); + } + } + Tuple minMaxPrimaryTermTuple = new Tuple<>(minPrimaryTem, maxPrimaryTem); + oldFormatMetadataFilePrimaryTermMap.put(metadataFile, minMaxPrimaryTermTuple); + return minMaxPrimaryTermTuple; } - Optional minPrimaryTerm = metadataFiles.stream() - .map(file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[1])) - .min(Long::compareTo); - // First we delete all stale primary terms folders from remote store - long minimumReferencedPrimaryTerm = minPrimaryTerm.get() - 1; - translogTransferManager.deletePrimaryTermsAsync(minimumReferencedPrimaryTerm); } } + + public static void cleanup(TranslogTransferManager translogTransferManager) throws IOException { + ActionListener> listMetadataFilesListener = new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + + try { + if (metadataFiles.isEmpty()) { + staticLogger.debug("No stale translog metadata files found"); + return; + } + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), staticLogger); + if (metadataFilesToBeDeleted.isEmpty()) { + staticLogger.debug("No metadata files to delete"); + return; + } + staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + + // For all the files that we are keeping, fetch min and max generations + List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + + // Delete stale metadata files + translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); + + // Delete stale primary terms + deleteStaleRemotePrimaryTerms( + metadataFilesNotToBeDeleted, + translogTransferManager, + new HashMap<>(), + new AtomicLong(Long.MAX_VALUE), + staticLogger + ); + } catch (Exception e) { + staticLogger.error("Exception while cleaning up metadata and primary terms", e); + } + } + + @Override + public void onFailure(Exception e) { + staticLogger.error("Exception while cleaning up metadata and primary terms", e); + } + }; + translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 56a9aa6447dec..291218ea47499 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -545,6 +545,14 @@ public void onFailure(Exception e) { }); } + public Set listPrimaryTermsInRemote() throws IOException { + Set primaryTermsStr = transferService.listFolders(remoteDataTransferPath); + if (primaryTermsStr != null) { + return primaryTermsStr.stream().map(Long::parseLong).collect(Collectors.toSet()); + } + return new HashSet<>(); + } + /** * Handles deletion of all translog files associated with a primary term. * diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 745fa9a8a219a..3b8885055e8f7 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * The metadata associated with every transfer {@link TransferSnapshot}. The metadata is uploaded at the end of the @@ -108,11 +109,28 @@ public String getFileName() { RemoteStoreUtils.invertLong(createdAt), String.valueOf(Objects.hash(nodeId)), RemoteStoreUtils.invertLong(minTranslogGeneration), + String.valueOf(getMinPrimaryTermReferred()), String.valueOf(CURRENT_VERSION) ) ); } + private long getMinPrimaryTermReferred() { + if (generationToPrimaryTermMapper.get() == null || generationToPrimaryTermMapper.get().values().isEmpty()) { + return -1; + } + Optional minPrimaryTerm = generationToPrimaryTermMapper.get() + .values() + .stream() + .map(s -> Long.parseLong(s)) + .min(Long::compareTo); + if (minPrimaryTerm.isPresent()) { + return minPrimaryTerm.get(); + } else { + return -1; + } + } + public static Tuple, String> getNodeIdByPrimaryTermAndGeneration(String filename) { String[] tokens = filename.split(METADATA_SEPARATOR); if (tokens.length < 6) { @@ -143,15 +161,43 @@ public static Tuple getMinMaxTranslogGenerationFromFilename(String f assert Version.CURRENT.onOrAfter(Version.V_2_17_0); try { // instead of direct index, we go backwards to avoid running into same separator in nodeId - String minGeneration = tokens[tokens.length - 2]; + String minGeneration = tokens[tokens.length - 3]; String maxGeneration = tokens[2]; return new Tuple<>(RemoteStoreUtils.invertLong(minGeneration), RemoteStoreUtils.invertLong(maxGeneration)); - } catch (NumberFormatException e) { + } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Exception while getting min and max translog generation from: {}", filename), e); return null; } } + public static Tuple getMinMaxPrimaryTermFromFilename(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + if (tokens.length < 7) { + // For versions < 2.17, we don't have min primary term. + return null; + } + assert Version.CURRENT.onOrAfter(Version.V_2_17_0); + try { + // instead of direct index, we go backwards to avoid running into same separator in nodeId + String minPrimaryTerm = tokens[tokens.length - 2]; + String maxPrimaryTerm = tokens[1]; + return new Tuple<>(Long.parseLong(minPrimaryTerm), RemoteStoreUtils.invertLong(maxPrimaryTerm)); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Exception while getting min and max primary term from: {}", filename), e); + return null; + } + } + + public static long getPrimaryTermFromFileName(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + try { + return RemoteStoreUtils.invertLong(tokens[1]); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Exception while getting max primary term from: {}", filename), e); + return -1; + } + } + @Override public int hashCode() { return Objects.hash(primaryTerm, generation); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 9ce9c0823429a..01d924aa17839 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -116,6 +116,7 @@ import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput; import org.opensearch.index.remote.RemoteStorePathStrategy.SnapshotShardPathInput; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.remote.RemoteTranslogTransferTracker; import org.opensearch.index.snapshots.IndexShardRestoreFailedException; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; @@ -132,6 +133,10 @@ import org.opensearch.index.store.lockmanager.FileLockInfo; import org.opensearch.index.store.lockmanager.RemoteStoreLockManager; import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory; +import org.opensearch.index.translog.RemoteFsTimestampAwareTranslog; +import org.opensearch.index.translog.RemoteFsTranslog; +import org.opensearch.index.translog.transfer.FileTransferTracker; +import org.opensearch.index.translog.transfer.TranslogTransferManager; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; @@ -2143,25 +2148,35 @@ private void cleanRemoteStoreDirectoryIfNeeded( } IndexMetadata prevIndexMetadata = this.getSnapshotIndexMetaData(oldRepoData, snapshotId, indexId); if (prevIndexMetadata != null && !isIndexPresent(clusterService, prevIndexMetadata.getIndexUUID())) { - String remoteStoreRepository = IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get( + String remoteStoreRepository = IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get( prevIndexMetadata.getSettings() ); assert (remoteStoreRepository != null); + String remoteTranslogRepositoryName = IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get( + prevIndexMetadata.getSettings() + ); + assert (remoteTranslogRepositoryName != null); + Repository remoteTranslogRepository = remoteSegmentStoreDirectoryFactory.getRepositoriesService() + .get() + .repository(remoteTranslogRepositoryName); + RemoteStorePathStrategy remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy( prevIndexMetadata ); for (int shardId = 0; shardId < prevIndexMetadata.getNumberOfShards(); shardId++) { + ShardId shard = new ShardId(Index.UNKNOWN_INDEX_NAME, prevIndexMetadata.getIndexUUID(), shardId); remoteDirectoryCleanupAsync( remoteSegmentStoreDirectoryFactory, threadPool, remoteStoreRepository, prevIndexMetadata.getIndexUUID(), - new ShardId(Index.UNKNOWN_INDEX_NAME, prevIndexMetadata.getIndexUUID(), shardId), + shard, ThreadPool.Names.REMOTE_PURGE, remoteStorePathStrategy ); + remoteTranslogCleanupAsync(remoteTranslogRepository, shard, remoteStorePathStrategy, prevIndexMetadata); } } } catch (Exception e) { @@ -2181,6 +2196,33 @@ private void cleanRemoteStoreDirectoryIfNeeded( } + private void remoteTranslogCleanupAsync( + Repository remoteTranslogRepository, + ShardId shardId, + RemoteStorePathStrategy remoteStorePathStrategy, + IndexMetadata prevIndexMetadata + ) { + assert remoteTranslogRepository instanceof BlobStoreRepository; + boolean indexMetadataEnabled = RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata); + RemoteTranslogTransferTracker remoteTranslogTransferTracker = new RemoteTranslogTransferTracker(shardId, 1000); + FileTransferTracker fileTransferTracker = new FileTransferTracker(shardId, remoteTranslogTransferTracker); + TranslogTransferManager translogTransferManager = RemoteFsTranslog.buildTranslogTransferManager( + (BlobStoreRepository) remoteTranslogRepository, + threadPool, + shardId, + fileTransferTracker, + remoteTranslogTransferTracker, + remoteStorePathStrategy, + remoteStoreSettings, + indexMetadataEnabled + ); + try { + RemoteFsTimestampAwareTranslog.cleanup(translogTransferManager); + } catch (IOException e) { + logger.error("Exception while cleaning up remote translog for shard: " + shardId, e); + } + } + /** * Finds and returns a list of shard paths that match the given index ID. * diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index 2a34dd3580948..be30de97ee830 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -637,7 +637,7 @@ private Tuple, Set> testGetPinnedTimestampLockedFilesW String metadataPrefix = "metadata__1__2__3__4__5__"; Map metadataFiles = new HashMap<>(); for (Long metadataFileTimestamp : metadataFileTimestamps) { - metadataFiles.put(metadataFileTimestamp, metadataPrefix + RemoteStoreUtils.invertLong(metadataFileTimestamp)); + metadataFiles.put(metadataFileTimestamp, metadataPrefix + RemoteStoreUtils.invertLong(metadataFileTimestamp) + "__1"); } return new Tuple<>( metadataFiles, @@ -662,7 +662,7 @@ private Tuple, Set> testGetPinnedTimestampLockedFilesW String primaryTerm = RemoteStoreUtils.invertLong(metadataFileTimestampPrimaryTerm.getValue()); String metadataPrefix = "metadata__" + primaryTerm + "__2__3__4__5__"; long metadataFileTimestamp = metadataFileTimestampPrimaryTerm.getKey(); - metadataFiles.put(metadataFileTimestamp, metadataPrefix + RemoteStoreUtils.invertLong(metadataFileTimestamp)); + metadataFiles.put(metadataFileTimestamp, metadataPrefix + RemoteStoreUtils.invertLong(metadataFileTimestamp) + "__1"); } return new Tuple<>( metadataFiles, diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 336d4bafd4b66..ecd6620dbea15 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -1170,9 +1170,9 @@ public void testInitializeToSpecificTimestampNoMdMatchingTimestamp() throws IOEx public void testInitializeToSpecificTimestampMatchingMdFile() throws IOException { String metadataPrefix = "metadata__1__2__3__4__5__"; List metadataFiles = new ArrayList<>(); - metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(1000)); - metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000)); - metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000)); + metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(1000) + "__1"); + metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(2000) + "__1"); + metadataFiles.add(metadataPrefix + RemoteStoreUtils.invertLong(3000) + "__1"); Map metadata = new HashMap<>(); metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major); @@ -1184,7 +1184,7 @@ public void testInitializeToSpecificTimestampMatchingMdFile() throws IOException Integer.MAX_VALUE ) ).thenReturn(metadataFiles); - when(remoteMetadataDirectory.getBlobStream(metadataPrefix + RemoteStoreUtils.invertLong(1000))).thenReturn( + when(remoteMetadataDirectory.getBlobStream(metadataPrefix + RemoteStoreUtils.invertLong(1000) + "__1")).thenReturn( createMetadataFileBytes(metadata, indexShard.getLatestReplicationCheckpoint(), segmentInfos) ); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 1f82dd9d7e641..c510a6475147d 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -8,6 +8,8 @@ package org.opensearch.index.translog; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -53,6 +55,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -63,7 +66,9 @@ import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -211,16 +216,44 @@ public void onFailure(Exception e) { // Node id containing separator String nodeIdWithSeparator = - "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node__1__9223372036438563958__1"; + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node__1__9223372036438563958__2__1"; Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(nodeIdWithSeparator); Long minGen = Long.MAX_VALUE - 9223372036438563958L; assertEquals(minGen, minMaxGen.v1()); // Malformed md filename - String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__1"; + String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__3__1"; assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(malformedMdFileName)); } + public void testGetMinMaxPrimaryTermFromFilename() throws Exception { + // New format metadata file + String newFormatMetadataFile = + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1"; + Tuple minMaxPrimaryterm = TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(newFormatMetadataFile); + Long minPrimaryTerm = 2L; + Long maxPrimaryTerm = 7L; + assertEquals(minPrimaryTerm, minMaxPrimaryterm.v1()); + assertEquals(maxPrimaryTerm, minMaxPrimaryterm.v2()); + + // Old format metadata file + String oldFormatMdFilename = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"; + assertNull(TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(oldFormatMdFilename)); + + // Node id containing separator + String nodeIdWithSeparator = + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node__1__9223372036438563958__2__1"; + minMaxPrimaryterm = TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(nodeIdWithSeparator); + minPrimaryTerm = 2L; + maxPrimaryTerm = 7L; + assertEquals(minPrimaryTerm, minMaxPrimaryterm.v1()); + assertEquals(maxPrimaryTerm, minMaxPrimaryterm.v2()); + + // Malformed md filename + String malformedMdFileName = "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__xyz__3qwe__1"; + assertNull(TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(malformedMdFileName)); + } + public void testIndexDeletionWithNoPinnedTimestampNoRecentMdFiles() throws Exception { RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); ArrayList ops = new ArrayList<>(); @@ -604,11 +637,11 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro List metadataFilesNotToBeDeleted = new ArrayList<>(); List metadataFilesToBeDeleted = List.of( // 4 to 7 - "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + "metadata__9223372036854775806__9223372036854775800__9223370311919910398__31__9223372036854775803__1__1", // 17 to 37 - "metadata__9223372036438563903__9223372036854775770__9223370311919910398__31__9223372036854775790__1", + "metadata__9223372036854775806__9223372036854775770__9223370311919910398__31__9223372036854775790__1__1", // 27 to 42 - "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" + "metadata__9223372036854775806__9223372036854775765__9223370311919910403__31__9223372036854775780__1__1" ); Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, @@ -618,6 +651,7 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro Set md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet()); Set md3Generations = LongStream.rangeClosed(27, 42).boxed().collect(Collectors.toSet()); + assertTrue(generations.containsAll(md1Generations)); assertTrue(generations.containsAll(md2Generations)); assertTrue(generations.containsAll(md3Generations)); @@ -631,19 +665,19 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro public void testGetGenerationsToBeDeleted() throws IOException { List metadataFilesNotToBeDeleted = List.of( // 1 to 4 - "metadata__9223372036438563903__9223372036854775803__9223370311919910398__31__9223372036854775806__1", + "metadata__9223372036854775806__9223372036854775803__9223370311919910398__31__9223372036854775806__1__1", // 26 to 30 - "metadata__9223372036438563903__9223372036854775777__9223370311919910398__31__9223372036854775781__1", + "metadata__9223372036854775806__9223372036854775777__9223370311919910398__31__9223372036854775781__1__1", // 42 to 100 - "metadata__9223372036438563903__9223372036854775707__9223370311919910403__31__9223372036854775765__1" + "metadata__9223372036854775806__9223372036854775707__9223370311919910403__31__9223372036854775765__1__1" ); List metadataFilesToBeDeleted = List.of( // 4 to 7 - "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + "metadata__9223372036854775806__9223372036854775800__9223370311919910398__31__9223372036854775803__1__1", // 17 to 37 - "metadata__9223372036438563903__9223372036854775770__9223370311919910398__31__9223372036854775790__1", + "metadata__9223372036854775806__9223372036854775770__9223370311919910398__31__9223372036854775790__1__1", // 27 to 42 - "metadata__9223372036438563903__9223372036854775765__9223370311919910403__31__9223372036854775780__1" + "metadata__9223372036854775806__9223372036854775765__9223370311919910403__31__9223372036854775780__1__1" ); Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, @@ -653,6 +687,7 @@ public void testGetGenerationsToBeDeleted() throws IOException { Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); Set md3Generations = LongStream.rangeClosed(31, 41).boxed().collect(Collectors.toSet()); + assertTrue(generations.containsAll(md1Generations)); assertTrue(generations.containsAll(md2Generations)); assertTrue(generations.containsAll(md3Generations)); @@ -783,49 +818,49 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException assertEquals( new Tuple<>(701L, 1008L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1", translogTransferManager ) ); assertEquals( new Tuple<>(4L, 7L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__2__1", translogTransferManager ) ); assertEquals( new Tuple<>(106L, 106L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1", + "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__3__1", translogTransferManager ) ); assertEquals( new Tuple<>(4573L, 99964L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854675843__9223370311919910408__31__9223372036854771234__1", + "metadata__9223372036438563903__9223372036854675843__9223370311919910408__31__9223372036854771234__4__1", translogTransferManager ) ); assertEquals( new Tuple<>(1L, 4L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854775803__9223370311919910413__31__9223372036854775806__1", + "metadata__9223372036438563903__9223372036854775803__9223370311919910413__31__9223372036854775806__5__1", translogTransferManager ) ); assertEquals( new Tuple<>(2474L, 3462L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854772345__9223370311919910429__31__9223372036854773333__1", + "metadata__9223372036438563903__9223372036854772345__9223370311919910429__31__9223372036854773333__6__1", translogTransferManager ) ); assertEquals( new Tuple<>(5807L, 7917L), translog.getMinMaxTranslogGenerationFromMetadataFile( - "metadata__9223372036438563903__9223372036854767890__9223370311919910434__31__9223372036854770000__1", + "metadata__9223372036438563903__9223372036854767890__9223370311919910434__31__9223372036854770000__7__1", translogTransferManager ) ); @@ -859,4 +894,93 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"); verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"); } + + public void testDeleteStaleRemotePrimaryTerms() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__9223372036438563958__4__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L)); + AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager).deletePrimaryTermsAsync(2L); + assertEquals(2, minPrimaryTermInRemote.get()); + + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + // This means there are no new invocations of deletePrimaryTermAsync + verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong()); + } + + public void testDeleteStaleRemotePrimaryTermsNoPrimaryTermInRemote() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__9223372036438563958__4__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of()); + AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager, times(0)).deletePrimaryTermsAsync(anyLong()); + assertEquals(Long.MAX_VALUE, minPrimaryTermInRemote.get()); + } + + public void testDeleteStaleRemotePrimaryTermsPrimaryTermInRemoteIsBigger() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__9223372036438563958__4__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__9223372036438563958__2__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(2L, 3L, 4L)); + AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager, times(0)).deletePrimaryTermsAsync(anyLong()); + assertEquals(2, minPrimaryTermInRemote.get()); + } + } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index 8605043ddd5b5..ed0d6b7d50706 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -628,7 +628,7 @@ public void testMetadataConflict() throws InterruptedException { String mdFilename = tm.getFileName(); long count = mdFilename.chars().filter(ch -> ch == METADATA_SEPARATOR.charAt(0)).count(); // There should not be any `_` in mdFile name as it is used a separator . - assertEquals(12, count); + assertEquals(14, count); Thread.sleep(1); TranslogTransferMetadata tm2 = new TranslogTransferMetadata(1, 1, 1, 2, "node--2"); String mdFilename2 = tm2.getFileName(); From b34543955eb8ef6faee78c08bae44bcec095e636 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Sep 2024 14:36:29 -0700 Subject: [PATCH 052/357] [RW Separation] Introduce allocation filter to control placement of search only replicas (#15455) * Introduce allocation filter to control placement of search only replicas Signed-off-by: Marc Handalian * Add a new decider rather than updating the existing FilterAllocationDecider Signed-off-by: Marc Handalian * Fix license header and description on SearchReplicaAllocationDecider Signed-off-by: Marc Handalian * Pr feedback. Signed-off-by: Marc Handalian * Fix class name to pass precommit checks Signed-off-by: Marc Handalian * Refactor all search replica create/update tests to a single OpenSearchSingleNodeTestCase. Signed-off-by: Marc Handalian * remove changelog entry Signed-off-by: Marc Handalian --------- Signed-off-by: Marc Handalian --- .../SearchReplicaFilteringAllocationIT.java | 125 ++++++++ .../SearchOnlyReplicaFeatureFlagIT.java | 12 + .../org/opensearch/cluster/ClusterModule.java | 5 + .../SearchReplicaAllocationDecider.java | 99 ++++++ .../common/settings/ClusterSettings.java | 5 +- .../MetadataCreateIndexServiceTests.java | 68 ---- .../metadata/SearchOnlyReplicaTests.java | 295 ++++++++++-------- .../decider/FilterAllocationDeciderTests.java | 2 +- .../SearchReplicaAllocationDeciderTests.java | 133 ++++++++ 9 files changed, 546 insertions(+), 198 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/cluster/allocation/SearchReplicaFilteringAllocationIT.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDecider.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDeciderTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/SearchReplicaFilteringAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/SearchReplicaFilteringAllocationIT.java new file mode 100644 index 0000000000000..5f65d6647f26d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/SearchReplicaFilteringAllocationIT.java @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.allocation; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SearchReplicaFilteringAllocationIT extends OpenSearchIntegTestCase { + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build(); + } + + public void testSearchReplicaDedicatedIncludes() { + List nodesIds = internalCluster().startNodes(3); + final String node_0 = nodesIds.get(0); + final String node_1 = nodesIds.get(1); + final String node_2 = nodesIds.get(2); + assertEquals(3, cluster().size()); + + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings( + Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node_1 + "," + node_0) + ) + .execute() + .actionGet(); + + createIndex( + "test", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build() + ); + ensureGreen("test"); + // ensure primary is not on node 0 or 1, + IndexShardRoutingTable routingTable = getRoutingTable(); + assertEquals(node_2, getNodeName(routingTable.primaryShard().currentNodeId())); + + String existingSearchReplicaNode = getNodeName(routingTable.searchOnlyReplicas().get(0).currentNodeId()); + String emptyAllowedNode = existingSearchReplicaNode.equals(node_0) ? node_1 : node_0; + + // set the included nodes to the other open node, search replica should relocate to that node. + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", emptyAllowedNode)) + .execute() + .actionGet(); + ensureGreen("test"); + + routingTable = getRoutingTable(); + assertEquals(node_2, getNodeName(routingTable.primaryShard().currentNodeId())); + assertEquals(emptyAllowedNode, getNodeName(routingTable.searchOnlyReplicas().get(0).currentNodeId())); + } + + public void testSearchReplicaDedicatedIncludes_DoNotAssignToOtherNodes() { + List nodesIds = internalCluster().startNodes(3); + final String node_0 = nodesIds.get(0); + final String node_1 = nodesIds.get(1); + final String node_2 = nodesIds.get(2); + assertEquals(3, cluster().size()); + + // set filter on 1 node and set search replica count to 2 - should leave 1 unassigned + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node_1)) + .execute() + .actionGet(); + + logger.info("--> creating an index with no replicas"); + createIndex( + "test", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 2) + .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build() + ); + ensureYellowAndNoInitializingShards("test"); + IndexShardRoutingTable routingTable = getRoutingTable(); + assertEquals(2, routingTable.searchOnlyReplicas().size()); + List assignedSearchShards = routingTable.searchOnlyReplicas() + .stream() + .filter(ShardRouting::assignedToNode) + .collect(Collectors.toList()); + assertEquals(1, assignedSearchShards.size()); + assertEquals(node_1, getNodeName(assignedSearchShards.get(0).currentNodeId())); + assertEquals(1, routingTable.searchOnlyReplicas().stream().filter(ShardRouting::unassigned).count()); + } + + private IndexShardRoutingTable getRoutingTable() { + IndexShardRoutingTable routingTable = getClusterState().routingTable().index("test").getShards().get(0); + return routingTable; + } + + private String getNodeName(String id) { + return getClusterState().nodes().get(id).getName(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java index e5a05c04fa7ee..ef18cff7e5b29 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java @@ -17,6 +17,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) public class SearchOnlyReplicaFeatureFlagIT extends OpenSearchIntegTestCase { @@ -53,4 +54,15 @@ public void testUpdateFeatureFlagDisabled() { }); assertTrue(settingsException.getMessage().contains("unknown setting")); } + + public void testFilterAllocationSettingNotRegistered() { + expectThrows(SettingsException.class, () -> { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", "node")) + .execute() + .actionGet(); + }); + } } diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index bb51c42252448..d9bb87a517927 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -75,6 +75,7 @@ import org.opensearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; +import org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.TargetPoolAllocationDecider; @@ -85,6 +86,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.core.ParseField; @@ -379,6 +381,9 @@ public static Collection createAllocationDeciders( addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider()); addAllocationDecider(deciders, new RestoreInProgressAllocationDecider()); addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings)); + if (FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.get(settings)) { + addAllocationDecider(deciders, new SearchReplicaAllocationDecider(settings, clusterSettings)); + } addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings)); addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings)); addAllocationDecider(deciders, new ThrottlingAllocationDecider(settings, clusterSettings)); diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDecider.java new file mode 100644 index 0000000000000..955c396bee4da --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDecider.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation.decider; + +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeFilters; +import org.opensearch.cluster.routing.RoutingNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; +import org.opensearch.common.settings.Settings; +import org.opensearch.node.remotestore.RemoteStoreNodeService; + +import java.util.Map; + +import static org.opensearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR; +import static org.opensearch.cluster.node.DiscoveryNodeFilters.OpType.OR; + +/** + * This allocation decider is similar to FilterAllocationDecider but provides + * the option to filter specifically for search replicas. + * The filter behaves similar to an include for any defined node attribute. + * A search replica can be allocated to only nodes with one of the specified attributes while + * other shard types will be rejected from nodes with any othe attributes. + * @opensearch.internal + */ +public class SearchReplicaAllocationDecider extends AllocationDecider { + + public static final String NAME = "filter"; + private static final String SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_PREFIX = "cluster.routing.allocation.search.replica.dedicated.include"; + public static final Setting.AffixSetting SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING = Setting.prefixKeySetting( + SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_PREFIX + ".", + key -> Setting.simpleString(key, value -> IP_VALIDATOR.accept(key, value), Property.Dynamic, Property.NodeScope) + ); + + private volatile DiscoveryNodeFilters searchReplicaIncludeFilters; + + private volatile RemoteStoreNodeService.Direction migrationDirection; + private volatile RemoteStoreNodeService.CompatibilityMode compatibilityMode; + + public SearchReplicaAllocationDecider(Settings settings, ClusterSettings clusterSettings) { + setSearchReplicaIncludeFilters(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING.getAsMap(settings)); + clusterSettings.addAffixMapUpdateConsumer( + SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING, + this::setSearchReplicaIncludeFilters, + (a, b) -> {} + ); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(shardRouting, node.node(), allocation); + } + + @Override + public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(shardRouting, node.node(), allocation); + } + + private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) { + if (searchReplicaIncludeFilters != null) { + final boolean match = searchReplicaIncludeFilters.match(node); + if (match == false && shardRouting.isSearchOnly()) { + return allocation.decision( + Decision.NO, + NAME, + "node does not match shard setting [%s] filters [%s]", + SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_PREFIX, + searchReplicaIncludeFilters + ); + } + // filter will only apply to search replicas + if (shardRouting.isSearchOnly() == false && match) { + return allocation.decision( + Decision.NO, + NAME, + "only search replicas can be allocated to node with setting [%s] filters [%s]", + SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_PREFIX, + searchReplicaIncludeFilters + ); + } + } + return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters"); + } + + private void setSearchReplicaIncludeFilters(Map filters) { + searchReplicaIncludeFilters = DiscoveryNodeFilters.trimTier( + DiscoveryNodeFilters.buildOrUpdateFromKeyValue(searchReplicaIncludeFilters, OR, filters) + ); + } +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index aef026f86c48d..552e15ab1b4c4 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -76,6 +76,7 @@ import org.opensearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; +import org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.opensearch.cluster.service.ClusterApplierService; @@ -813,6 +814,8 @@ public void apply(Settings value, Settings current, Settings previous) { OpenSearchOnHeapCacheSettings.EXPIRE_AFTER_ACCESS_SETTING.getConcreteSettingForNamespace( CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() ) - ) + ), + List.of(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL), + List.of(SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING) ); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index d376d0db8d6f4..3f223706819b7 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -137,12 +137,10 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SEARCH_REPLICAS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_REPLICATION_TYPE_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; @@ -156,7 +154,6 @@ import static org.opensearch.cluster.metadata.MetadataCreateIndexService.getIndexNumberOfRoutingShards; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.parseV1Mappings; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases; -import static org.opensearch.common.util.FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL; import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.opensearch.index.IndexSettings.INDEX_MERGE_POLICY; @@ -2507,71 +2504,6 @@ public void testApplyContextWithSettingsOverlap() throws IOException { } } - public void testDefaultSearchReplicasSetting() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build()); - Settings templateSettings = Settings.EMPTY; - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder(); - request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - templateSettings, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - clusterSettings - ); - assertFalse(INDEX_NUMBER_OF_SEARCH_REPLICAS_SETTING.exists(indexSettings)); - } - - public void testSearchReplicasValidationWithSegmentReplication() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build()); - Settings templateSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 2); - request.settings(requestSettings.build()); - Settings indexSettings = aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - templateSettings, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - clusterSettings - ); - assertEquals("2", indexSettings.get(SETTING_NUMBER_OF_SEARCH_REPLICAS)); - assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); - } - - public void testSearchReplicasValidationWithDocumentReplication() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, Boolean.TRUE).build()); - Settings templateSettings = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build(); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - final Settings.Builder requestSettings = Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 2); - request.settings(requestSettings.build()); - - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - templateSettings, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet(), - clusterSettings - ) - ); - assertEquals("To set index.number_of_search_only_replicas, index.replication.type must be set to SEGMENT", exception.getMessage()); - } - private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/SearchOnlyReplicaTests.java b/server/src/test/java/org/opensearch/cluster/metadata/SearchOnlyReplicaTests.java index b1dd397c97218..3d11193a07884 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/SearchOnlyReplicaTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/SearchOnlyReplicaTests.java @@ -24,9 +24,11 @@ import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.cluster.ClusterStateChanges; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; import java.util.ArrayList; import java.util.Collections; @@ -40,157 +42,194 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.common.util.FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL; -public class SearchOnlyReplicaTests extends OpenSearchTestCase { +public class SearchOnlyReplicaTests extends OpenSearchSingleNodeTestCase { - public void testUpdateSearchReplicaCount() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, "true").build()); - final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + private ThreadPool threadPool; + + @Before + public void setUp() throws Exception { + super.setUp(); + this.threadPool = new TestThreadPool(getClass().getName()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + terminate(threadPool); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.getKey(), true) + .build(); + } + + public void testCreateWithDefaultSearchReplicasSetting() { final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool); + ClusterState state = createIndexWithSettings(cluster, Settings.builder().build()); + IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); + assertEquals(1, indexShardRoutingTable.replicaShards().size()); + assertEquals(0, indexShardRoutingTable.searchOnlyReplicas().size()); + assertEquals(1, indexShardRoutingTable.writerReplicas().size()); + } - try { - List allNodes = new ArrayList<>(); - // node for primary/local - DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); - allNodes.add(localNode); - // node for search replicas - we'll start with 1 and add another - for (int i = 0; i < 2; i++) { - allNodes.add(createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)); - } - ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); - - CreateIndexRequest request = new CreateIndexRequest( - "index", + public void testSearchReplicasValidationWithDocumentReplication() { + final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool); + RuntimeException exception = expectThrows( + RuntimeException.class, + () -> createIndexWithSettings( + cluster, Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) .build() - ).waitForActiveShards(ActiveShardCount.NONE); - state = cluster.createIndex(state, request); - assertTrue(state.metadata().hasIndex("index")); - rerouteUntilActive(state, cluster); - IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); - assertEquals(1, indexShardRoutingTable.replicaShards().size()); - assertEquals(1, indexShardRoutingTable.searchOnlyReplicas().size()); - assertEquals(0, indexShardRoutingTable.writerReplicas().size()); - - // add another replica - state = cluster.updateSettings( - state, - new UpdateSettingsRequest("index").settings(Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 2).build()) - ); - rerouteUntilActive(state, cluster); - indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); - assertEquals(2, indexShardRoutingTable.replicaShards().size()); - assertEquals(2, indexShardRoutingTable.searchOnlyReplicas().size()); - assertEquals(0, indexShardRoutingTable.writerReplicas().size()); - - // remove all replicas - state = cluster.updateSettings( - state, - new UpdateSettingsRequest("index").settings(Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0).build()) - ); - rerouteUntilActive(state, cluster); - indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); - assertEquals(0, indexShardRoutingTable.replicaShards().size()); - assertEquals(0, indexShardRoutingTable.searchOnlyReplicas().size()); - assertEquals(0, indexShardRoutingTable.writerReplicas().size()); - } finally { - terminate(threadPool); - } + ) + ); + assertEquals( + "To set index.number_of_search_only_replicas, index.replication.type must be set to SEGMENT", + exception.getCause().getMessage() + ); } - public void testUpdateSearchReplicasOverShardLimit() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, "true").build()); - final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + public void testUpdateSearchReplicaCount() { final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool); - try { - List allNodes = new ArrayList<>(); - // node for primary/local - DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); - allNodes.add(localNode); + ClusterState state = createIndexWithSettings( + cluster, + Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build() + ); + assertTrue(state.metadata().hasIndex("index")); + rerouteUntilActive(state, cluster); + IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); + assertEquals(1, indexShardRoutingTable.replicaShards().size()); + assertEquals(1, indexShardRoutingTable.searchOnlyReplicas().size()); + assertEquals(0, indexShardRoutingTable.writerReplicas().size()); + + // add another replica + state = cluster.updateSettings( + state, + new UpdateSettingsRequest("index").settings(Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 2).build()) + ); + rerouteUntilActive(state, cluster); + indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); + assertEquals(2, indexShardRoutingTable.replicaShards().size()); + assertEquals(2, indexShardRoutingTable.searchOnlyReplicas().size()); + assertEquals(0, indexShardRoutingTable.writerReplicas().size()); + + // remove all replicas + state = cluster.updateSettings( + state, + new UpdateSettingsRequest("index").settings(Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0).build()) + ); + rerouteUntilActive(state, cluster); + indexShardRoutingTable = state.getRoutingTable().index("index").getShards().get(0); + assertEquals(0, indexShardRoutingTable.replicaShards().size()); + assertEquals(0, indexShardRoutingTable.searchOnlyReplicas().size()); + assertEquals(0, indexShardRoutingTable.writerReplicas().size()); + } + private ClusterState createIndexWithSettings(ClusterStateChanges cluster, Settings settings) { + List allNodes = new ArrayList<>(); + // node for primary/local + DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); + allNodes.add(localNode); + // node for search replicas - we'll start with 1 and add another + for (int i = 0; i < 2; i++) { allNodes.add(createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)); + } + ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); - ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); + CreateIndexRequest request = new CreateIndexRequest("index", settings).waitForActiveShards(ActiveShardCount.NONE); + state = cluster.createIndex(state, request); + return state; + } - CreateIndexRequest request = new CreateIndexRequest( - "index", - Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) - .build() - ).waitForActiveShards(ActiveShardCount.NONE); - state = cluster.createIndex(state, request); - assertTrue(state.metadata().hasIndex("index")); - rerouteUntilActive(state, cluster); - - // add another replica - ClusterState finalState = state; - Integer maxShardPerNode = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getDefault(Settings.EMPTY); - expectThrows( - RuntimeException.class, - () -> cluster.updateSettings( - finalState, - new UpdateSettingsRequest("index").settings( - Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, maxShardPerNode * 2).build() - ) - ) - ); + public void testUpdateSearchReplicasOverShardLimit() { + final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool); - } finally { - terminate(threadPool); - } + List allNodes = new ArrayList<>(); + // node for primary/local + DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); + allNodes.add(localNode); + + allNodes.add(createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)); + + ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); + + CreateIndexRequest request = new CreateIndexRequest( + "index", + Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build() + ).waitForActiveShards(ActiveShardCount.NONE); + state = cluster.createIndex(state, request); + assertTrue(state.metadata().hasIndex("index")); + rerouteUntilActive(state, cluster); + + // add another replica + ClusterState finalState = state; + Integer maxShardPerNode = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getDefault(Settings.EMPTY); + expectThrows( + RuntimeException.class, + () -> cluster.updateSettings( + finalState, + new UpdateSettingsRequest("index").settings( + Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, maxShardPerNode * 2).build() + ) + ) + ); } public void testUpdateSearchReplicasOnDocrepCluster() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(READER_WRITER_SPLIT_EXPERIMENTAL, "true").build()); - final ThreadPool threadPool = new TestThreadPool(getClass().getName()); final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool); - try { - List allNodes = new ArrayList<>(); - // node for primary/local - DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); - allNodes.add(localNode); - - allNodes.add(createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)); - - ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); - - CreateIndexRequest request = new CreateIndexRequest( - "index", - Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) - .build() - ).waitForActiveShards(ActiveShardCount.NONE); - state = cluster.createIndex(state, request); - assertTrue(state.metadata().hasIndex("index")); - rerouteUntilActive(state, cluster); - - // add another replica - ClusterState finalState = state; - Integer maxShardPerNode = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getDefault(Settings.EMPTY); - expectThrows( - RuntimeException.class, - () -> cluster.updateSettings( - finalState, - new UpdateSettingsRequest("index").settings( - Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, maxShardPerNode * 2).build() - ) + List allNodes = new ArrayList<>(); + // node for primary/local + DiscoveryNode localNode = createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE); + allNodes.add(localNode); + + allNodes.add(createNode(Version.CURRENT, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)); + + ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); + + CreateIndexRequest request = new CreateIndexRequest( + "index", + Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) + .build() + ).waitForActiveShards(ActiveShardCount.NONE); + state = cluster.createIndex(state, request); + assertTrue(state.metadata().hasIndex("index")); + rerouteUntilActive(state, cluster); + + // add another replica + ClusterState finalState = state; + Integer maxShardPerNode = ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getDefault(Settings.EMPTY); + expectThrows( + RuntimeException.class, + () -> cluster.updateSettings( + finalState, + new UpdateSettingsRequest("index").settings( + Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, maxShardPerNode * 2).build() ) - ); - } finally { - terminate(threadPool); - } + ) + ); + } private static void rerouteUntilActive(ClusterState state, ClusterStateChanges cluster) { diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java index 2e303887e0f1b..f226c45553d57 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java @@ -323,7 +323,7 @@ private ClusterState createInitialClusterState(AllocationService service, Settin return createInitialClusterState(service, indexSettings, Settings.EMPTY); } - private ClusterState createInitialClusterState(AllocationService service, Settings idxSettings, Settings clusterSettings) { + static ClusterState createInitialClusterState(AllocationService service, Settings idxSettings, Settings clusterSettings) { Metadata.Builder metadata = Metadata.builder(); metadata.persistentSettings(clusterSettings); final Settings.Builder indexSettings = settings(Version.CURRENT).put(idxSettings); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDeciderTests.java new file mode 100644 index 0000000000000..8d4f4cdee26cc --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/SearchReplicaAllocationDeciderTests.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation.decider; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.EmptyClusterInfoService; +import org.opensearch.cluster.OpenSearchAllocationTestCase; +import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.UnassignedInfo; +import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.snapshots.EmptySnapshotsInfoService; +import org.opensearch.test.gateway.TestGatewayAllocator; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import static org.opensearch.cluster.routing.allocation.decider.SearchReplicaAllocationDecider.SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING; + +public class SearchReplicaAllocationDeciderTests extends OpenSearchAllocationTestCase { + + public void testSearchReplicaRoutingDedicatedIncludes() { + // we aren't using a settingsModule here so we need to set feature flag gated setting + Set> settings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + settings.add(SEARCH_REPLICA_ROUTING_INCLUDE_GROUP_SETTING); + ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), settings); + Settings initialSettings = Settings.builder() + .put("cluster.routing.allocation.search.replica.dedicated.include._id", "node1,node2") + .build(); + + SearchReplicaAllocationDecider filterAllocationDecider = new SearchReplicaAllocationDecider(initialSettings, clusterSettings); + AllocationDeciders allocationDeciders = new AllocationDeciders( + Arrays.asList( + filterAllocationDecider, + new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), + new ReplicaAfterPrimaryActiveAllocationDecider() + ) + ); + AllocationService service = new AllocationService( + allocationDeciders, + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + EmptyClusterInfoService.INSTANCE, + EmptySnapshotsInfoService.INSTANCE + ); + ClusterState state = FilterAllocationDeciderTests.createInitialClusterState(service, Settings.EMPTY, Settings.EMPTY); + RoutingTable routingTable = state.routingTable(); + RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state, null, null, 0); + allocation.debugDecision(true); + + ShardRouting searchReplica = ShardRouting.newUnassigned( + routingTable.index("sourceIndex").shard(0).shardId(), + false, + true, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "") + ); + + ShardRouting regularReplica = ShardRouting.newUnassigned( + routingTable.index("sourceIndex").shard(0).shardId(), + false, + false, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "") + ); + + ShardRouting primary = ShardRouting.newUnassigned( + routingTable.index("sourceIndex").shard(0).shardId(), + true, + false, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "") + ); + + Decision.Single decision = (Decision.Single) filterAllocationDecider.canAllocate( + searchReplica, + state.getRoutingNodes().node("node2"), + allocation + ); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(searchReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + + decision = (Decision.Single) filterAllocationDecider.canAllocate(regularReplica, state.getRoutingNodes().node("node2"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(regularReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + + decision = (Decision.Single) filterAllocationDecider.canAllocate(primary, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(primary, state.getRoutingNodes().node("node2"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + + Settings updatedSettings = Settings.builder() + .put("cluster.routing.allocation.search.replica.dedicated.include._id", "node2") + .build(); + clusterSettings.applySettings(updatedSettings); + + decision = (Decision.Single) filterAllocationDecider.canAllocate(searchReplica, state.getRoutingNodes().node("node2"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(searchReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canRemain(searchReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + + decision = (Decision.Single) filterAllocationDecider.canAllocate(regularReplica, state.getRoutingNodes().node("node2"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(regularReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canRemain(regularReplica, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + + decision = (Decision.Single) filterAllocationDecider.canAllocate(primary, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canAllocate(primary, state.getRoutingNodes().node("node2"), allocation); + assertEquals(decision.toString(), Decision.Type.NO, decision.type()); + decision = (Decision.Single) filterAllocationDecider.canRemain(primary, state.getRoutingNodes().node("node1"), allocation); + assertEquals(decision.toString(), Decision.Type.YES, decision.type()); + } +} From d64baa6808a14fa021b16972459257b43ac6b7da Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 4 Sep 2024 15:22:06 -0700 Subject: [PATCH 053/357] Add new cluster setting for keyword indexordocvalues query (#15637) * Add new cluster setting for keyword indexordocvalues query Signed-off-by: Harsha Vamsi Kalluri * Fix tests Signed-off-by: Harsha Vamsi Kalluri --------- Signed-off-by: Harsha Vamsi Kalluri --- CHANGELOG.md | 1 + .../common/settings/ClusterSettings.java | 1 + .../org/opensearch/index/IndexService.java | 8 ++- .../index/mapper/KeywordFieldMapper.java | 15 +++++ .../index/query/QueryShardContext.java | 66 ++++++++++++++++++- .../search/DefaultSearchContext.java | 18 ++++- .../org/opensearch/search/SearchService.java | 8 +++ .../search/internal/SearchContext.java | 5 ++ .../index/mapper/KeywordFieldTypeTests.java | 8 +-- .../index/search/NestedHelperTests.java | 10 +-- .../search/DefaultSearchContextTests.java | 20 +++--- .../index/mapper/FieldTypeTestCase.java | 8 ++- 12 files changed, 138 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71868de93812e..8b35ba2613c3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) +- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 552e15ab1b4c4..136eeb4e33251 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -550,6 +550,7 @@ public void apply(Settings value, Settings current, Settings previous) { SearchService.MAX_AGGREGATION_REWRITE_FILTERS, SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING, SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD, + SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED, CreatePitController.PIT_INIT_KEEP_ALIVE, Node.WRITE_PORTS_FILE_SETTING, Node.NODE_NAME_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 12b02d3dbd6fa..5121fb79207ed 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -853,7 +853,7 @@ public IndexSettings getIndexSettings() { * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { - return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, false); + return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, false, false); } /** @@ -867,7 +867,8 @@ public QueryShardContext newQueryShardContext( IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias, - boolean validate + boolean validate, + boolean keywordIndexOrDocValuesEnabled ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher( index().getName(), @@ -893,7 +894,8 @@ public QueryShardContext newQueryShardContext( indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, - validate + validate, + keywordIndexOrDocValuesEnabled ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 2116ac522b705..11ff601b3fd6d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -392,6 +392,9 @@ public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); // has index and doc_values enabled if (isSearchable() && hasDocValues()) { + if (!context.keywordFieldIndexOrDocValuesEnabled()) { + return super.termsQuery(values, context); + } BytesRef[] bytesRefs = new BytesRef[values.size()]; for (int i = 0; i < bytesRefs.length; i++) { bytesRefs[i] = indexedValueForSearch(values.get(i)); @@ -429,6 +432,9 @@ public Query prefixQuery( } failIfNotIndexedAndNoDocValues(); if (isSearchable() && hasDocValues()) { + if (!context.keywordFieldIndexOrDocValuesEnabled()) { + return super.prefixQuery(value, method, caseInsensitive, context); + } Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); return new IndexOrDocValuesQuery(indexQuery, dvQuery); @@ -461,6 +467,9 @@ public Query regexpQuery( } failIfNotIndexedAndNoDocValues(); if (isSearchable() && hasDocValues()) { + if (!context.keywordFieldIndexOrDocValuesEnabled()) { + return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + } Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); Query dvQuery = super.regexpQuery( value, @@ -549,6 +558,9 @@ public Query fuzzyQuery( ); } if (isSearchable() && hasDocValues()) { + if (!context.keywordFieldIndexOrDocValuesEnabled()) { + return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + } Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); Query dvQuery = super.fuzzyQuery( value, @@ -591,6 +603,9 @@ public Query wildcardQuery( // wildcard // query text if (isSearchable() && hasDocValues()) { + if (!context.keywordFieldIndexOrDocValuesEnabled()) { + return super.wildcardQuery(value, method, caseInsensitive, true, context); + } Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); return new IndexOrDocValuesQuery(indexQuery, dvQuery); diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index 91313092d8d28..bccead2b029d0 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -125,6 +125,7 @@ public class QueryShardContext extends QueryRewriteContext { private final ValuesSourceRegistry valuesSourceRegistry; private BitSetProducer parentFilter; private DerivedFieldResolver derivedFieldResolver; + private boolean keywordIndexOrDocValuesEnabled; public QueryShardContext( int shardId, @@ -208,7 +209,55 @@ public QueryShardContext( ), allowExpensiveQueries, valuesSourceRegistry, - validate + validate, + false + ); + } + + public QueryShardContext( + int shardId, + IndexSettings indexSettings, + BigArrays bigArrays, + BitsetFilterCache bitsetFilterCache, + TriFunction, IndexFieldData> indexFieldDataLookup, + MapperService mapperService, + SimilarityService similarityService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + boolean validate, + boolean keywordIndexOrDocValuesEnabled + ) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + validate, + keywordIndexOrDocValuesEnabled ); } @@ -231,7 +280,8 @@ public QueryShardContext(QueryShardContext source) { source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, - source.validate() + source.validate(), + source.keywordIndexOrDocValuesEnabled ); } @@ -253,7 +303,8 @@ private QueryShardContext( Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry, - boolean validate + boolean validate, + boolean keywordIndexOrDocValuesEnabled ) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis, validate); this.shardId = shardId; @@ -277,6 +328,7 @@ private QueryShardContext( emptyList(), indexSettings.isDerivedFieldAllowed() ); + this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled; } private void reset() { @@ -414,6 +466,14 @@ public void setDerivedFieldResolver(DerivedFieldResolver derivedFieldResolver) { this.derivedFieldResolver = derivedFieldResolver; } + public boolean keywordFieldIndexOrDocValuesEnabled() { + return keywordIndexOrDocValuesEnabled; + } + + public void setKeywordFieldIndexOrDocValuesEnabled(boolean keywordIndexOrDocValuesEnabled) { + this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled; + } + public void setAllowUnmappedFields(boolean allowUnmappedFields) { this.allowUnmappedFields = allowUnmappedFields; } diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index fa762f9183d2a..1706c27ccf922 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -120,6 +120,7 @@ import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO; import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE; +import static org.opensearch.search.SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED; import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS; /** @@ -206,6 +207,7 @@ final class DefaultSearchContext extends SearchContext { private final SetOnce requestShouldUseConcurrentSearch = new SetOnce<>(); private final int maxAggRewriteFilters; private final int cardinalityAggregationPruningThreshold; + private final boolean keywordIndexOrDocValuesEnabled; DefaultSearchContext( ReaderContext readerContext, @@ -256,7 +258,8 @@ final class DefaultSearchContext extends SearchContext { this.searcher, request::nowInMillis, shardTarget.getClusterAlias(), - validate + validate, + evaluateKeywordIndexOrDocValuesEnabled() ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; @@ -265,6 +268,7 @@ final class DefaultSearchContext extends SearchContext { this.maxAggRewriteFilters = evaluateFilterRewriteSetting(); this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold(); this.concurrentSearchDeciders = concurrentSearchDeciders; + this.keywordIndexOrDocValuesEnabled = evaluateKeywordIndexOrDocValuesEnabled(); } @Override @@ -1117,10 +1121,22 @@ public int cardinalityAggregationPruningThreshold() { return cardinalityAggregationPruningThreshold; } + @Override + public boolean keywordIndexOrDocValuesEnabled() { + return keywordIndexOrDocValuesEnabled; + } + private int evaluateCardinalityAggregationPruningThreshold() { if (clusterService != null) { return clusterService.getClusterSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD); } return 0; } + + public boolean evaluateKeywordIndexOrDocValuesEnabled() { + if (clusterService != null) { + return clusterService.getClusterSettings().get(KEYWORD_INDEX_OR_DOC_VALUES_ENABLED); + } + return false; + } } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 39c0c19a978e5..e2a804a674d8f 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -337,6 +337,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv Property.NodeScope ); + public static final Setting KEYWORD_INDEX_OR_DOC_VALUES_ENABLED = Setting.boolSetting( + "search.keyword_index_or_doc_values_enabled", + false, + Property.Dynamic, + Property.NodeScope + ); + public static final int DEFAULT_SIZE = 10; public static final int DEFAULT_FROM = 0; @@ -1173,6 +1180,7 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear context.getIndexSettings().isDerivedFieldAllowed() && allowDerivedField ); context.setDerivedFieldResolver(derivedFieldResolver); + context.setKeywordFieldIndexOrDocValuesEnabled(searchContext.keywordIndexOrDocValuesEnabled()); searchContext.getQueryShardContext().setDerivedFieldResolver(derivedFieldResolver); Rewriteable.rewrite(request.getRewriteable(), context, true); assert searchContext.getQueryShardContext().isCacheable(); diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index bc4b7058651dd..5357206e8c117 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -526,4 +526,9 @@ public int maxAggRewriteFilters() { public int cardinalityAggregationPruningThreshold() { return 0; } + + public boolean keywordIndexOrDocValuesEnabled() { + return false; + } + } diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java index b10035f54a0c0..f291b864beb59 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java @@ -136,7 +136,7 @@ public void testTermsQuery() { new TermInSetQuery("field", terms), new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms) ); - assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null)); + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); Query expectedIndex = new TermInSetQuery("field", terms); @@ -225,7 +225,7 @@ public void testRegexpQuery() { new RegexpQuery(new Term("field", "foo.*")), new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE) ), - ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); Query indexExpected = new RegexpQuery(new Term("field", "foo.*")); @@ -267,7 +267,7 @@ public void testFuzzyQuery() { new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true), new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) ), - ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC) + ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) ); Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true); @@ -308,7 +308,7 @@ public void testWildCardQuery() { MultiTermQuery.DOC_VALUES_REWRITE ) ); - assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); + assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); Query indexExpected = new WildcardQuery(new Term("field", new BytesRef("foo*"))); MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); diff --git a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java index 7ffcc0fb7437a..f7f921e824490 100644 --- a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java @@ -57,6 +57,8 @@ import java.io.IOException; import java.util.Collections; +import static org.opensearch.index.mapper.FieldTypeTestCase.MOCK_QSC_ENABLE_INDEX_DOC_VALUES; + public class NestedHelperTests extends OpenSearchSingleNodeTestCase { IndexService indexService; @@ -132,28 +134,28 @@ public void testMatchNo() { } public void testTermsQuery() { - Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), null); + Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES); assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); - termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), null); + termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES); assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); - termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), null); + termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES); assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing")); - termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), null); + termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES); assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery)); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1")); assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2")); diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 491a0377ab32e..7e213218eb97b 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -168,9 +168,8 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queryShardContext); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); @@ -501,9 +500,8 @@ public void testClearQueryCancellationsOnClose() throws IOException { IndexService indexService = mock(IndexService.class); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queryShardContext); Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) @@ -598,9 +596,8 @@ public void testSearchPathEvaluation() throws Exception { IndexService indexService = mock(IndexService.class); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queryShardContext); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); @@ -830,9 +827,8 @@ public void testSearchPathEvaluationWithConcurrentSearchModeAsAuto() throws Exce IndexService indexService = mock(IndexService.class); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queryShardContext); IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/FieldTypeTestCase.java index 7ed0da8509fab..5d85844f3218d 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/FieldTypeTestCase.java @@ -46,16 +46,18 @@ /** Base test case for subclasses of MappedFieldType */ public abstract class FieldTypeTestCase extends OpenSearchTestCase { - public static final QueryShardContext MOCK_QSC = createMockQueryShardContext(true); - public static final QueryShardContext MOCK_QSC_DISALLOW_EXPENSIVE = createMockQueryShardContext(false); + public static final QueryShardContext MOCK_QSC = createMockQueryShardContext(true, false); + public static final QueryShardContext MOCK_QSC_DISALLOW_EXPENSIVE = createMockQueryShardContext(false, false); + public static final QueryShardContext MOCK_QSC_ENABLE_INDEX_DOC_VALUES = createMockQueryShardContext(true, true); protected QueryShardContext randomMockShardContext() { return randomFrom(MOCK_QSC, MOCK_QSC_DISALLOW_EXPENSIVE); } - static QueryShardContext createMockQueryShardContext(boolean allowExpensiveQueries) { + static QueryShardContext createMockQueryShardContext(boolean allowExpensiveQueries, boolean keywordIndexOrDocValuesEnabled) { QueryShardContext queryShardContext = mock(QueryShardContext.class); when(queryShardContext.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); + when(queryShardContext.keywordFieldIndexOrDocValuesEnabled()).thenReturn(keywordIndexOrDocValuesEnabled); return queryShardContext; } From 729e40dcc6f5f90b9acba8d9809baf9842ac5938 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Sep 2024 18:16:23 -0700 Subject: [PATCH 054/357] Fix bwc test failures by updating wire compatibility on NodeIndicesStats (#15709) --- .../action/admin/indices/stats/CommonStatsFlags.java | 4 ++-- .../main/java/org/opensearch/indices/NodeIndicesStats.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java index 04f39d77ce6c8..03fb55323feec 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java @@ -101,7 +101,7 @@ public CommonStatsFlags(StreamInput in) throws IOException { includeCaches = in.readEnumSet(CacheType.class); levels = in.readStringArray(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { includeIndicesStatsByLevel = in.readBoolean(); } } @@ -128,7 +128,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnumSet(includeCaches); out.writeStringArrayNullable(levels); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeBoolean(includeIndicesStatsByLevel); } } diff --git a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java index 83a759cdb71c5..4c28c08d8061b 100644 --- a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java @@ -83,7 +83,7 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment { public NodeIndicesStats(StreamInput in) throws IOException { stats = new CommonStats(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { // contains statsByIndex if (in.readBoolean()) { statsByIndex = readStatsByIndex(in); @@ -284,7 +284,7 @@ public RecoveryStats getRecoveryStats() { public void writeTo(StreamOutput out) throws IOException { stats.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeBoolean(statsByIndex != null); if (statsByIndex != null) { writeStatsByIndex(out); From 261b17a4fb9978fc2a9ef82958efe5d7ff63ffa6 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Sep 2024 20:04:55 -0700 Subject: [PATCH 055/357] Mute test testRemoteStoreCleanupForDeletedIndexForSnapshotV2 (#15695) Signed-off-by: Marc Handalian --- .../java/org/opensearch/snapshots/DeleteSnapshotV2IT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index 44d5c0a28cd9a..1d7a58384c0be 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -206,6 +206,7 @@ public void testDeleteShallowCopyV2MultipleSnapshots() throws Exception { } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15692") public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); From 375c0bf1c9ac9ff7226d447bcd1dddfcedd856fc Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 4 Sep 2024 20:20:32 -0700 Subject: [PATCH 056/357] [RW Separation] Add polling segment replication for search replicas (#15627) --- CHANGELOG.md | 2 +- .../SearchReplicaReplicationIT.java | 85 +++++++++ .../org/opensearch/index/IndexModule.java | 54 +++++- .../org/opensearch/index/IndexService.java | 70 +++++++- .../index/seqno/ReplicationTracker.java | 11 +- .../opensearch/indices/IndicesService.java | 9 +- .../SegmentReplicationTargetService.java | 1 + .../replication/SegmentReplicator.java | 35 +++- .../main/java/org/opensearch/node/Node.java | 7 +- .../opensearch/index/IndexModuleTests.java | 3 +- .../opensearch/index/IndexServiceTests.java | 53 ++++++ .../replication/SegmentReplicatorTests.java | 163 ++++++++++++++++++ 12 files changed, 478 insertions(+), 15 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java create mode 100644 server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b35ba2613c3c..d1c0c78d6db02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) - Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) - Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) -- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410)) +- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237)) - [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) - Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java new file mode 100644 index 0000000000000..a1b512c326ac5 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaReplicationIT.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.nio.file.Path; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SearchReplicaReplicationIT extends SegmentReplicationBaseIT { + + private static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; + + private Boolean useRemoteStore; + + @Before + public void randomizeRemoteStoreEnabled() { + useRemoteStore = randomBoolean(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + if (useRemoteStore) { + if (absolutePath == null) { + absolutePath = randomRepoPath().toAbsolutePath(); + } + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .build(); + } + return super.nodeSettings(nodeOrdinal); + } + + @After + public void teardown() { + if (useRemoteStore) { + clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get(); + } + } + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build(); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, true).build(); + } + + public void testReplication() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + final String primary = internalCluster().startDataOnlyNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startDataOnlyNode(); + ensureGreen(INDEX_NAME); + + final int docCount = 10; + for (int i = 0; i < docCount; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().get(); + } + refresh(INDEX_NAME); + waitForSearchableDocs(docCount, primary, replica); + } + +} diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index dc1bf94662385..79de97dc96fba 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -73,6 +73,7 @@ import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.IndexEventListener; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexingOperationListener; import org.opensearch.index.shard.SearchOperationListener; import org.opensearch.index.similarity.SimilarityService; @@ -629,6 +630,56 @@ public IndexService newIndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings + ) throws IOException { + return newIndexService( + indexCreationContext, + environment, + xContentRegistry, + shardStoreDeleter, + circuitBreakerService, + bigArrays, + threadPool, + scriptService, + clusterService, + client, + indicesQueryCache, + mapperRegistry, + indicesFieldDataCache, + namedWriteableRegistry, + idFieldDataEnabled, + valuesSourceRegistry, + remoteDirectoryFactory, + translogFactorySupplier, + clusterDefaultRefreshIntervalSupplier, + recoverySettings, + remoteStoreSettings, + (s) -> {} + ); + } + + public IndexService newIndexService( + IndexService.IndexCreationContext indexCreationContext, + NodeEnvironment environment, + NamedXContentRegistry xContentRegistry, + IndexService.ShardStoreDeleter shardStoreDeleter, + CircuitBreakerService circuitBreakerService, + BigArrays bigArrays, + ThreadPool threadPool, + ScriptService scriptService, + ClusterService clusterService, + Client client, + IndicesQueryCache indicesQueryCache, + MapperRegistry mapperRegistry, + IndicesFieldDataCache indicesFieldDataCache, + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled, + ValuesSourceRegistry valuesSourceRegistry, + IndexStorePlugin.DirectoryFactory remoteDirectoryFactory, + BiFunction translogFactorySupplier, + Supplier clusterDefaultRefreshIntervalSupplier, + RecoverySettings recoverySettings, + RemoteStoreSettings remoteStoreSettings, + Consumer replicator ) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = indexReaderWrapper @@ -689,7 +740,8 @@ public IndexService newIndexService( recoverySettings, remoteStoreSettings, fileCache, - compositeIndexSettings + compositeIndexSettings, + replicator ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 5121fb79207ed..e5dd44a70a11c 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -136,6 +136,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.opensearch.common.collect.MapBuilder.newMapBuilder; +import static org.opensearch.common.util.FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING; import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdater.indexHasRemoteStoreSettings; /** @@ -174,6 +175,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private volatile AsyncTranslogFSync fsyncTask; private volatile AsyncGlobalCheckpointTask globalCheckpointTask; private volatile AsyncRetentionLeaseSyncTask retentionLeaseSyncTask; + private volatile AsyncReplicationTask asyncReplicationTask; // don't convert to Setting<> and register... we only set this in tests and register via a plugin private final String INDEX_TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING = "index.translog.retention.check_interval"; @@ -194,6 +196,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final RemoteStoreSettings remoteStoreSettings; private final FileCache fileCache; private final CompositeIndexSettings compositeIndexSettings; + private final Consumer replicator; public IndexService( IndexSettings indexSettings, @@ -231,7 +234,8 @@ public IndexService( RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, FileCache fileCache, - CompositeIndexSettings compositeIndexSettings + CompositeIndexSettings compositeIndexSettings, + Consumer replicator ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -306,11 +310,15 @@ public IndexService( this.trimTranslogTask = new AsyncTrimTranslogTask(this); this.globalCheckpointTask = new AsyncGlobalCheckpointTask(this); this.retentionLeaseSyncTask = new AsyncRetentionLeaseSyncTask(this); + if (READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.get(indexSettings.getNodeSettings())) { + this.asyncReplicationTask = new AsyncReplicationTask(this); + } this.translogFactorySupplier = translogFactorySupplier; this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; + this.replicator = replicator; updateFsyncTaskIfNecessary(); } @@ -386,7 +394,8 @@ public IndexService( recoverySettings, remoteStoreSettings, null, - null + null, + s -> {} ); } @@ -395,6 +404,11 @@ static boolean needsMapperService(IndexSettings indexSettings, IndexCreationCont && indexCreationContext == IndexCreationContext.CREATE_INDEX); // metadata verification needs a mapper service } + // visible for tests + AsyncReplicationTask getReplicationTask() { + return asyncReplicationTask; + } + /** * Context for index creation * @@ -1067,11 +1081,22 @@ public synchronized void updateMetadata(final IndexMetadata currentIndexMetadata } onRefreshIntervalChange(); updateFsyncTaskIfNecessary(); + if (READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.get(indexSettings.getNodeSettings())) { + updateReplicationTask(); + } } metadataListeners.forEach(c -> c.accept(newIndexMetadata)); } + private void updateReplicationTask() { + try { + asyncReplicationTask.close(); + } finally { + asyncReplicationTask = new AsyncReplicationTask(this); + } + } + /** * Called whenever the refresh interval changes. This can happen in 2 cases - * 1. {@code cluster.default.index.refresh_interval} cluster setting changes. The change would only happen for @@ -1336,6 +1361,47 @@ public String toString() { } } + final class AsyncReplicationTask extends BaseAsyncTask { + + AsyncReplicationTask(IndexService indexService) { + super(indexService, indexService.getRefreshInterval()); + } + + @Override + protected void runInternal() { + indexService.maybeSyncSegments(false); + } + + @Override + protected String getThreadPool() { + return ThreadPool.Names.GENERIC; + } + + @Override + public String toString() { + return "replication"; + } + + @Override + protected boolean mustReschedule() { + return indexSettings.isSegRepEnabledOrRemoteNode() && super.mustReschedule(); + } + } + + private void maybeSyncSegments(boolean force) { + if (getRefreshInterval().millis() > 0 || force) { + for (IndexShard shard : this.shards.values()) { + try { + if (shard.routingEntry().isSearchOnly() && shard.routingEntry().active()) { + replicator.accept(shard); + } + } catch (IndexShardClosedException | AlreadyClosedException ex) { + // do nothing + } + } + } + } + final class AsyncTrimTranslogTask extends BaseAsyncTask { AsyncTrimTranslogTask(IndexService indexService) { diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index b9cb5e92d0ed1..1e43827afeb47 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -1251,12 +1251,13 @@ public ReplicationCheckpoint getLatestReplicationCheckpoint() { return this.latestReplicationCheckpoint; } - private boolean isPrimaryRelocation(String allocationId) { + // skip any shard that is a relocating primary or search only replica (not tracked by primary) + private boolean shouldSkipReplicationTimer(String allocationId) { Optional shardRouting = routingTable.shards() .stream() .filter(routing -> routing.allocationId().getId().equals(allocationId)) .findAny(); - return shardRouting.isPresent() && shardRouting.get().primary(); + return shardRouting.isPresent() && (shardRouting.get().primary() || shardRouting.get().isSearchOnly()); } private void createReplicationLagTimers() { @@ -1268,7 +1269,7 @@ private void createReplicationLagTimers() { // it is possible for a shard to be in-sync but not yet removed from the checkpoints collection after a failover event. if (cps.inSync && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false - && isPrimaryRelocation(allocationId) == false + && shouldSkipReplicationTimer(allocationId) == false && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint) && (indexSettings.isSegRepLocalEnabled() == true || isShardOnRemoteEnabledNode.apply(routingTable.getByAllocationId(allocationId).currentNodeId()))) { @@ -1302,7 +1303,7 @@ public synchronized void startReplicationLagTimers(ReplicationCheckpoint checkpo final CheckpointState cps = e.getValue(); if (cps.inSync && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false - && isPrimaryRelocation(e.getKey()) == false + && shouldSkipReplicationTimer(e.getKey()) == false && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint) && cps.checkpointTimers.containsKey(latestReplicationCheckpoint)) { cps.checkpointTimers.get(latestReplicationCheckpoint).start(); @@ -1330,7 +1331,7 @@ public synchronized Set getSegmentReplicationStats entry -> entry.getKey().equals(this.shardAllocationId) == false && entry.getValue().inSync && replicationGroup.getUnavailableInSyncShards().contains(entry.getKey()) == false - && isPrimaryRelocation(entry.getKey()) == false + && shouldSkipReplicationTimer(entry.getKey()) == false /*Check if the current primary shard is migrating to remote and all the other shard copies of the same index still hasn't completely moved over to the remote enabled nodes. Ensures that: diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index be16d4ea184fa..4593aedfe1f83 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -359,6 +359,7 @@ public class IndicesService extends AbstractLifecycleComponent private final SearchRequestStats searchRequestStats; private final FileCache fileCache; private final CompositeIndexSettings compositeIndexSettings; + private final Consumer replicator; @Override protected void doStart() { @@ -395,7 +396,8 @@ public IndicesService( CacheService cacheService, RemoteStoreSettings remoteStoreSettings, FileCache fileCache, - CompositeIndexSettings compositeIndexSettings + CompositeIndexSettings compositeIndexSettings, + Consumer replicator ) { this.settings = settings; this.threadPool = threadPool; @@ -504,6 +506,7 @@ protected void closeInternal() { this.remoteStoreSettings = remoteStoreSettings; this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; + this.replicator = replicator; } public IndicesService( @@ -564,6 +567,7 @@ public IndicesService( cacheService, remoteStoreSettings, null, + null, null ); } @@ -980,7 +984,8 @@ private synchronized IndexService createIndexService( translogFactorySupplier, this::getClusterDefaultRefreshInterval, this.recoverySettings, - this.remoteStoreSettings + this.remoteStoreSettings, + replicator ); } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index a042e08c2a53f..8fee3f671ecc9 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -157,6 +157,7 @@ public SegmentReplicationTargetService( ForceSyncRequest::new, new ForceSyncTransportRequestHandler() ); + replicator.setSourceFactory(sourceFactory); } @Override diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java index 417556c22636e..ad3bc1933208c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicator.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; import org.opensearch.OpenSearchCorruptionException; +import org.opensearch.common.SetOnce; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; @@ -44,12 +45,44 @@ public class SegmentReplicator { private final Map completedReplications = ConcurrentCollections.newConcurrentMap(); private final ThreadPool threadPool; + private final SetOnce sourceFactory; + public SegmentReplicator(ThreadPool threadPool) { this.onGoingReplications = new ReplicationCollection<>(logger, threadPool); this.threadPool = threadPool; + this.sourceFactory = new SetOnce<>(); + } + + /** + * Starts a replication event for the given shard. + * @param shard - {@link IndexShard} replica shard + */ + public void startReplication(IndexShard shard) { + if (sourceFactory.get() == null) return; + startReplication( + shard, + shard.getLatestReplicationCheckpoint(), + sourceFactory.get().get(shard), + new SegmentReplicationTargetService.SegmentReplicationListener() { + @Override + public void onReplicationDone(SegmentReplicationState state) { + logger.trace("Completed replication for {}", shard.shardId()); + } + + @Override + public void onReplicationFailure(SegmentReplicationState state, ReplicationFailedException e, boolean sendShardFailure) { + logger.error(() -> new ParameterizedMessage("Failed segment replication for {}", shard.shardId()), e); + if (sendShardFailure) { + shard.failShard("unrecoverable replication failure", e); + } + } + } + ); } - // TODO: Add public entrypoint for replication on an interval to be invoked via IndexService + void setSourceFactory(SegmentReplicationSourceFactory sourceFactory) { + this.sourceFactory.set(sourceFactory); + } /** * Start a round of replication and sync to at least the given checkpoint. diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index c55c2ab683b11..b143333954338 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -897,6 +897,7 @@ protected Node( remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); CacheModule cacheModule = new CacheModule(pluginsService.filterPlugins(CachePlugin.class), settings); CacheService cacheService = cacheModule.getCacheService(); + final SegmentReplicator segmentReplicator = new SegmentReplicator(threadPool); final IndicesService indicesService = new IndicesService( settings, pluginsService, @@ -926,7 +927,8 @@ protected Node( cacheService, remoteStoreSettings, fileCache, - compositeIndexSettings + compositeIndexSettings, + segmentReplicator::startReplication ); final IngestService ingestService = new IngestService( @@ -1439,7 +1441,7 @@ protected Node( new SegmentReplicationSourceFactory(transportService, recoverySettings, clusterService), indicesService, clusterService, - new SegmentReplicator(threadPool) + segmentReplicator ) ); b.bind(SegmentReplicationSourceService.class) @@ -1474,6 +1476,7 @@ protected Node( b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry); b.bind(SegmentReplicationStatsTracker.class).toInstance(segmentReplicationStatsTracker); b.bind(SearchRequestOperationsCompositeListenerFactory.class).toInstance(searchRequestOperationsCompositeListenerFactory); + b.bind(SegmentReplicator.class).toInstance(segmentReplicator); taskManagerClientOptional.ifPresent(value -> b.bind(TaskManagerClient.class).toInstance(value)); }); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 8618d776bdf04..bd86d3d396987 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -264,7 +264,8 @@ private IndexService newIndexService(IndexModule module) throws IOException { translogFactorySupplier, () -> IndexSettings.DEFAULT_REFRESH_INTERVAL, DefaultRecoverySettings.INSTANCE, - DefaultRemoteStoreSettings.INSTANCE + DefaultRemoteStoreSettings.INSTANCE, + s -> {} ); } diff --git a/server/src/test/java/org/opensearch/index/IndexServiceTests.java b/server/src/test/java/org/opensearch/index/IndexServiceTests.java index 14451ef21726e..5905e64cede1b 100644 --- a/server/src/test/java/org/opensearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/opensearch/index/IndexServiceTests.java @@ -41,6 +41,7 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.MediaTypeRegistry; @@ -52,6 +53,7 @@ import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -591,6 +593,57 @@ public void testIndexSortBackwardCompatible() { } } + public void testReplicationTask() throws Exception { + // create with docrep - task should not schedule + IndexService indexService = createIndex( + "docrep_index", + Settings.builder().put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT).build() + ); + final Index index = indexService.index(); + ensureGreen(index.getName()); + IndexService.AsyncReplicationTask task = indexService.getReplicationTask(); + assertFalse(task.isScheduled()); + assertFalse(task.mustReschedule()); + + // create for segrep - task should schedule + indexService = createIndex( + "segrep_index", + Settings.builder() + .put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "5s") + .build() + ); + final Index srIndex = indexService.index(); + ensureGreen(srIndex.getName()); + task = indexService.getReplicationTask(); + assertTrue(task.isScheduled()); + assertTrue(task.mustReschedule()); + assertEquals(5000, task.getInterval().millis()); + + // test we can update the interval + client().admin() + .indices() + .prepareUpdateSettings("segrep_index") + .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s")) + .get(); + + IndexService.AsyncReplicationTask updatedTask = indexService.getReplicationTask(); + assertNotSame(task, updatedTask); + assertFalse(task.isScheduled()); + assertTrue(task.isClosed()); + assertTrue(updatedTask.isScheduled()); + assertTrue(updatedTask.mustReschedule()); + assertEquals(1000, updatedTask.getInterval().millis()); + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING.getKey(), true) + .build(); + } + private static String createTestMapping(String type) { return " \"properties\": {\n" + " \"test\": {\n" diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java new file mode 100644 index 0000000000000..7acee449a1b46 --- /dev/null +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.apache.lucene.store.IOContext; +import org.opensearch.OpenSearchCorruptionException; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.replication.TestReplicationSource; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.IndexShardTestCase; +import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.CopyState; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; + +import org.mockito.Mockito; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class SegmentReplicatorTests extends IndexShardTestCase { + + private static final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + + public void testStartReplicationWithoutSourceFactory() { + ThreadPool threadpool = mock(ThreadPool.class); + ExecutorService mock = mock(ExecutorService.class); + when(threadpool.generic()).thenReturn(mock); + SegmentReplicator segmentReplicator = new SegmentReplicator(threadpool); + + IndexShard shard = mock(IndexShard.class); + segmentReplicator.startReplication(shard); + Mockito.verifyNoInteractions(mock); + } + + public void testStartReplicationRunsSuccessfully() throws Exception { + final IndexShard replica = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + final IndexShard primary = newStartedShard(true, settings, new NRTReplicationEngineFactory()); + + // index and copy segments to replica. + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + indexDoc(primary, "_doc", Integer.toString(i)); + } + primary.refresh("test"); + + SegmentReplicator segmentReplicator = spy(new SegmentReplicator(threadPool)); + SegmentReplicationSourceFactory factory = mock(SegmentReplicationSourceFactory.class); + when(factory.get(replica)).thenReturn(new TestReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + resolveCheckpointListener(listener, primary); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + BiConsumer fileProgressTracker, + ActionListener listener + ) { + try { + Lucene.cleanLuceneIndex(indexShard.store().directory()); + Map segmentMetadataMap = primary.getSegmentMetadataMap(); + for (String file : segmentMetadataMap.keySet()) { + indexShard.store().directory().copyFrom(primary.store().directory(), file, file, IOContext.DEFAULT); + } + listener.onResponse(new GetSegmentFilesResponse(new ArrayList<>(segmentMetadataMap.values()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + segmentReplicator.setSourceFactory(factory); + segmentReplicator.startReplication(replica); + assertBusy(() -> assertDocCount(replica, numDocs)); + closeShards(primary, replica); + } + + public void testReplicationFails() throws Exception { + allowShardFailures(); + final IndexShard replica = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + final IndexShard primary = newStartedShard(true, settings, new NRTReplicationEngineFactory()); + + SegmentReplicator segmentReplicator = spy(new SegmentReplicator(threadPool)); + SegmentReplicationSourceFactory factory = mock(SegmentReplicationSourceFactory.class); + when(factory.get(replica)).thenReturn(new TestReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + resolveCheckpointListener(listener, primary); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + BiConsumer fileProgressTracker, + ActionListener listener + ) { + OpenSearchCorruptionException corruptIndexException = new OpenSearchCorruptionException("test"); + try { + indexShard.store().markStoreCorrupted(corruptIndexException); + } catch (IOException e) { + throw new RuntimeException(e); + } + listener.onFailure(corruptIndexException); + } + }); + // assert shard failure on corruption + AtomicBoolean failureCallbackTriggered = new AtomicBoolean(false); + replica.addShardFailureCallback((ig) -> failureCallbackTriggered.set(true)); + segmentReplicator.setSourceFactory(factory); + segmentReplicator.startReplication(replica); + assertBusy(() -> assertTrue(failureCallbackTriggered.get())); + closeShards(primary, replica); + } + + protected void resolveCheckpointListener(ActionListener listener, IndexShard primary) { + try (final CopyState copyState = new CopyState(primary)) { + listener.onResponse( + new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes()) + ); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} From 73e14ab1be3c7b37c061e16c5718331c965ec303 Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Thu, 5 Sep 2024 08:57:34 +0530 Subject: [PATCH 057/357] version change to V_2.17.0 for node min version check (#15644) Signed-off-by: Anshu Agarwal Co-authored-by: Anshu Agarwal --- .../main/java/org/opensearch/snapshots/SnapshotsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index c6585985fc82a..a2364d96f2098 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -288,7 +288,7 @@ public void executeSnapshot(final CreateSnapshotRequest request, final ActionLis if (remoteStoreIndexShallowCopy && isSnapshotV2 && request.indices().length == 0 - && clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.CURRENT)) { + && clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_2_17_0)) { createSnapshotV2(request, listener); } else { createSnapshot( From f33c786a1fbf44ee48e34674ed42fefd6a8d7dbd Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:22:44 +0530 Subject: [PATCH 058/357] Throw CircuitBreakingException for too many shards in snashot status API call (#15688) Signed-off-by: Lakshya Taragi --- .../snapshots/SnapshotStatusApisIT.java | 19 +++++++-------- .../TransportSnapshotsStatusAction.java | 23 +++++++++++++------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index 5a043e69e9735..c3214022df663 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -51,6 +51,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexNotFoundException; @@ -602,11 +603,11 @@ public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws // across a single snapshot assertBusy(() -> { - TooManyShardsInSnapshotsStatusException exception = expectThrows( - TooManyShardsInSnapshotsStatusException.class, + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, () -> client().admin().cluster().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot1).execute().actionGet() ); - assertEquals(exception.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); assertTrue( exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") ); @@ -614,8 +615,8 @@ public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws // across multiple snapshots assertBusy(() -> { - TooManyShardsInSnapshotsStatusException exception = expectThrows( - TooManyShardsInSnapshotsStatusException.class, + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, () -> client().admin() .cluster() .prepareSnapshotStatus(repositoryName) @@ -623,7 +624,7 @@ public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws .execute() .actionGet() ); - assertEquals(exception.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); assertTrue( exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") ); @@ -741,8 +742,8 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2)); assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - TooManyShardsInSnapshotsStatusException ex = expectThrows( - TooManyShardsInSnapshotsStatusException.class, + CircuitBreakingException ex = expectThrows( + CircuitBreakingException.class, () -> client().admin() .cluster() .prepareSnapshotStatus(repositoryName) @@ -751,7 +752,7 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { .execute() .actionGet() ); - assertEquals(ex.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE); + assertEquals(ex.status(), RestStatus.TOO_MANY_REQUESTS); assertTrue(ex.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request")); logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index f2a9b88f790c9..8228cb6301c8c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -49,6 +49,8 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.index.shard.ShardId; @@ -66,7 +68,6 @@ import org.opensearch.snapshots.SnapshotShardsService; import org.opensearch.snapshots.SnapshotState; import org.opensearch.snapshots.SnapshotsService; -import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -458,13 +459,17 @@ private Map snapshotsInfo( snapshotsInfoMap.put(snapshotId, snapshotInfo); } if (totalShardsAcrossSnapshots > maximumAllowedShardCount && request.indices().length == 0) { - String message = "Total shard count [" + String message = "[" + + repositoryName + + ":" + + String.join(", ", request.snapshots()) + + "]" + + " Total shard count [" + totalShardsAcrossSnapshots + "] is more than the maximum allowed value of shard count [" + maximumAllowedShardCount + "] for snapshot status request"; - - throw new TooManyShardsInSnapshotsStatusException(repositoryName, message, request.snapshots()); + throw new CircuitBreakingException(message, CircuitBreaker.Durability.PERMANENT); } return unmodifiableMap(snapshotsInfoMap); } @@ -520,15 +525,19 @@ private Map snapshotShards( } if (totalShardsAcrossIndices > maximumAllowedShardCount && requestedIndexNames.isEmpty() == false && isV2Snapshot == false) { - String message = "Total shard count [" + String message = "[" + + repositoryName + + ":" + + String.join(", ", request.snapshots()) + + "]" + + " Total shard count [" + totalShardsAcrossIndices + "] across the requested indices [" + requestedIndexNames.stream().collect(Collectors.joining(", ")) + "] is more than the maximum allowed value of shard count [" + maximumAllowedShardCount + "] for snapshot status request"; - - throw new TooManyShardsInSnapshotsStatusException(repositoryName, message, snapshotName); + throw new CircuitBreakingException(message, CircuitBreaker.Durability.PERMANENT); } final Map shardStatus = new HashMap<>(); From 4f50b4d705deaa971d802cd22fb120f75c722517 Mon Sep 17 00:00:00 2001 From: Rishab Nahata Date: Thu, 5 Sep 2024 11:36:03 +0530 Subject: [PATCH 059/357] Schedule reroute after allocator timed out (#15565) * Schedule reroute after allocator timed out Signed-off-by: Rishab Nahata --- .../org/opensearch/cluster/ClusterModule.java | 4 + .../allocator/BalancedShardsAllocator.java | 25 ++ .../allocation/allocator/ShardsAllocator.java | 3 + .../gateway/ShardsBatchGatewayAllocator.java | 30 +- .../main/java/org/opensearch/node/Node.java | 1 + .../cluster/ClusterModuleTests.java | 13 + ...TimeBoundBalancedShardsAllocatorTests.java | 257 +++++++++++++----- .../gateway/GatewayAllocatorTests.java | 62 ++++- .../TestShardBatchGatewayAllocator.java | 6 +- 9 files changed, 321 insertions(+), 80 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index d9bb87a517927..3924e8e0589a7 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -53,6 +53,7 @@ import org.opensearch.cluster.metadata.ViewMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.DelayedAllocationService; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator; import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; @@ -479,4 +480,7 @@ public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator, Shard allocationService.setExistingShardsAllocators(existingShardsAllocators); } + public void setRerouteServiceForAllocator(RerouteService rerouteService) { + shardsAllocator.setRerouteService(rerouteService); + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index a5193ca602f04..785636fa7ff2a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.IntroSorter; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.RoutingNode; import org.opensearch.cluster.routing.RoutingNodes; import org.opensearch.cluster.routing.ShardMovementStrategy; @@ -49,12 +50,14 @@ import org.opensearch.cluster.routing.allocation.RebalanceParameter; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.ShardAllocationDecision; +import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; import java.util.HashMap; import java.util.HashSet; @@ -202,6 +205,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { private volatile boolean ignoreThrottleInRestore; private volatile TimeValue allocatorTimeout; private long startTime; + private RerouteService rerouteService; public BalancedShardsAllocator(Settings settings) { this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); @@ -231,6 +235,12 @@ public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSetting clusterSettings.addSettingsUpdateConsumer(ALLOCATOR_TIMEOUT_SETTING, this::setAllocatorTimeout); } + @Override + public void setRerouteService(RerouteService rerouteService) { + assert this.rerouteService == null : "RerouteService is already set"; + this.rerouteService = rerouteService; + } + /** * Changes in deprecated setting SHARD_MOVE_PRIMARY_FIRST_SETTING affect value of its replacement setting SHARD_MOVEMENT_STRATEGY_SETTING. */ @@ -342,6 +352,7 @@ public void allocate(RoutingAllocation allocation) { localShardsBalancer.allocateUnassigned(); localShardsBalancer.moveShards(); localShardsBalancer.balance(); + scheduleRerouteIfAllocatorTimedOut(); final ShardsBalancer remoteShardsBalancer = new RemoteShardsBalancer(logger, allocation); remoteShardsBalancer.allocateUnassigned(); @@ -404,6 +415,20 @@ private void failAllocationOfNewPrimaries(RoutingAllocation allocation) { } } + private void scheduleRerouteIfAllocatorTimedOut() { + if (allocatorTimedOut()) { + assert rerouteService != null : "RerouteService not set to schedule reroute after allocator time out"; + rerouteService.reroute( + "reroute after balanced shards allocator timed out", + Priority.HIGH, + ActionListener.wrap( + r -> logger.trace("reroute after balanced shards allocator timed out completed"), + e -> logger.debug("reroute after balanced shards allocator timed out failed", e) + ) + ); + } + } + /** * Returns the currently configured delta threshold */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java index 29e9acca4e6c2..38aafff6ce3e8 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsAllocator.java @@ -32,6 +32,7 @@ package org.opensearch.cluster.routing.allocation.allocator; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision; import org.opensearch.cluster.routing.allocation.MoveDecision; @@ -73,4 +74,6 @@ public interface ShardsAllocator { * the cluster explain API, then this method should throw a {@code UnsupportedOperationException}. */ ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation); + + default void setRerouteService(RerouteService rerouteService) {} } diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java index d18304ea73ed0..5e2dcbcd70b40 100644 --- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java @@ -184,11 +184,11 @@ public void cleanCaches() { // for tests protected ShardsBatchGatewayAllocator() { - this(DEFAULT_SHARD_BATCH_SIZE); + this(DEFAULT_SHARD_BATCH_SIZE, null); } - protected ShardsBatchGatewayAllocator(long batchSize) { - this.rerouteService = null; + protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteService) { + this.rerouteService = rerouteService; this.batchStartedAction = null; this.primaryShardBatchAllocator = null; this.batchStoreAction = null; @@ -297,6 +297,18 @@ public void run() { public void onComplete() { logger.trace("Triggering oncomplete after timeout for [{}] primary shards", timedOutPrimaryShardIds.size()); primaryBatchShardAllocator.allocateUnassignedBatchOnTimeout(timedOutPrimaryShardIds, allocation, true); + if (timedOutPrimaryShardIds.isEmpty() == false) { + logger.trace("scheduling reroute after existing shards allocator timed out for primary shards"); + assert rerouteService != null; + rerouteService.reroute( + "reroute after existing shards allocator timed out", + Priority.HIGH, + ActionListener.wrap( + r -> logger.trace("reroute after existing shards allocator timed out completed"), + e -> logger.debug("reroute after existing shards allocator timed out failed", e) + ) + ); + } } }; } else { @@ -320,6 +332,18 @@ public void run() { public void onComplete() { logger.trace("Triggering oncomplete after timeout for [{}] replica shards", timedOutReplicaShardIds.size()); replicaBatchShardAllocator.allocateUnassignedBatchOnTimeout(timedOutReplicaShardIds, allocation, false); + if (timedOutReplicaShardIds.isEmpty() == false) { + logger.trace("scheduling reroute after existing shards allocator timed out for replica shards"); + assert rerouteService != null; + rerouteService.reroute( + "reroute after existing shards allocator timed out", + Priority.HIGH, + ActionListener.wrap( + r -> logger.trace("reroute after existing shards allocator timed out completed"), + e -> logger.debug("reroute after existing shards allocator timed out failed", e) + ) + ); + } } }; } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index b143333954338..34ed957542aff 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -871,6 +871,7 @@ protected Node( final RerouteService rerouteService = new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute); rerouteServiceReference.set(rerouteService); clusterService.setRerouteService(rerouteService); + clusterModule.setRerouteServiceForAllocator(rerouteService); final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java index 97706927ba857..f8240e775cfa5 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java @@ -337,6 +337,19 @@ public void testQueryGroupMetadataRegister() { ); } + public void testRerouteServiceSetForBalancedShardsAllocator() { + ClusterModule clusterModule = new ClusterModule( + Settings.EMPTY, + clusterService, + Collections.emptyList(), + clusterInfoService, + null, + threadContext, + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) + ); + clusterModule.setRerouteServiceForAllocator((reason, priority, listener) -> listener.onResponse(clusterService.state())); + } + private static ClusterPlugin existingShardsAllocatorPlugin(final String allocatorName) { return new ClusterPlugin() { @Override diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/allocator/TimeBoundBalancedShardsAllocatorTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/allocator/TimeBoundBalancedShardsAllocatorTests.java index a10c305686638..45a0bd7b18afd 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/allocator/TimeBoundBalancedShardsAllocatorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/allocator/TimeBoundBalancedShardsAllocatorTests.java @@ -8,6 +8,7 @@ package org.opensearch.cluster.routing.allocation.allocator; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.cluster.ClusterInfo; import org.opensearch.cluster.ClusterName; @@ -17,6 +18,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.RoutingNodes; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; @@ -26,14 +28,20 @@ import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.test.ClusterServiceUtils; +import org.opensearch.threadpool.TestThreadPool; +import org.junit.After; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.opensearch.cluster.routing.ShardRoutingState.STARTED; @@ -41,26 +49,49 @@ public class TimeBoundBalancedShardsAllocatorTests extends OpenSearchAllocationTestCase { + private TestThreadPool threadPool; + private ClusterService clusterService; + private ClusterState state; + private final DiscoveryNode node1 = newNode("node1", "node1", Collections.singletonMap("zone", "1a")); private final DiscoveryNode node2 = newNode("node2", "node2", Collections.singletonMap("zone", "1b")); private final DiscoveryNode node3 = newNode("node3", "node3", Collections.singletonMap("zone", "1c")); - public void testAllUnassignedShardsAllocatedWhenNoTimeOut() { + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + if (threadPool != null) { + final boolean terminated = terminate(threadPool); + assert terminated; + } + if (clusterService != null) { + clusterService.close(); + } + } + + public void setupStateAndService(Metadata metadata, RoutingTable routingTable) { + state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(routingTable) + .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) + .build(); + threadPool = new TestThreadPool(getTestName()); + clusterService = ClusterServiceUtils.createClusterService(state, threadPool); + } + + public void testAllUnassignedShardsAllocatedWhenNoTimeOutAndRerouteNotScheduled() { int numberOfIndices = 2; int numberOfShards = 5; int numberOfReplicas = 1; int totalPrimaryCount = numberOfIndices * numberOfShards; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Settings.Builder settings = Settings.builder(); - // passing total shard count for timed out latch such that no shard times out - BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings.build(), new CountDownLatch(totalShardCount)); + // passing sufficiently high count for timeout latch to simulate no time out + BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings.build(), new CountDownLatch(Integer.MAX_VALUE)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); RoutingAllocation allocation = new RoutingAllocation( yesAllocationDeciders(), new RoutingNodes(state, false), @@ -69,6 +100,18 @@ public void testAllUnassignedShardsAllocatedWhenNoTimeOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List initializingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING); int node1Recoveries = allocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()); @@ -77,9 +120,10 @@ public void testAllUnassignedShardsAllocatedWhenNoTimeOut() { assertEquals(totalShardCount, initializingShards.size()); assertEquals(0, allocation.routingNodes().unassigned().ignored().size()); assertEquals(totalPrimaryCount, node1Recoveries + node2Recoveries + node3Recoveries); + assertFalse(rerouteScheduled.get()); } - public void testAllUnassignedShardsIgnoredWhenTimedOut() { + public void testAllUnassignedShardsIgnoredWhenTimedOutAndRerouteScheduled() { int numberOfIndices = 2; int numberOfShards = 5; int numberOfReplicas = 1; @@ -89,11 +133,7 @@ public void testAllUnassignedShardsIgnoredWhenTimedOut() { BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings.build(), new CountDownLatch(0)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); RoutingAllocation allocation = new RoutingAllocation( yesAllocationDeciders(), new RoutingNodes(state, false), @@ -102,6 +142,18 @@ public void testAllUnassignedShardsIgnoredWhenTimedOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List initializingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING); int node1Recoveries = allocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()); @@ -110,9 +162,10 @@ public void testAllUnassignedShardsIgnoredWhenTimedOut() { assertEquals(0, initializingShards.size()); assertEquals(totalShardCount, allocation.routingNodes().unassigned().ignored().size()); assertEquals(0, node1Recoveries + node2Recoveries + node3Recoveries); + assertTrue(rerouteScheduled.get()); } - public void testAllocatePartialPrimaryShardsUntilTimedOut() { + public void testAllocatePartialPrimaryShardsUntilTimedOutAndRerouteScheduled() { int numberOfIndices = 2; int numberOfShards = 5; int numberOfReplicas = 1; @@ -123,11 +176,7 @@ public void testAllocatePartialPrimaryShardsUntilTimedOut() { BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings.build(), new CountDownLatch(shardsToAllocate)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); RoutingAllocation allocation = new RoutingAllocation( yesAllocationDeciders(), new RoutingNodes(state, false), @@ -136,6 +185,18 @@ public void testAllocatePartialPrimaryShardsUntilTimedOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List initializingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING); int node1Recoveries = allocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()); @@ -144,9 +205,10 @@ public void testAllocatePartialPrimaryShardsUntilTimedOut() { assertEquals(shardsToAllocate, initializingShards.size()); assertEquals(totalShardCount - shardsToAllocate, allocation.routingNodes().unassigned().ignored().size()); assertEquals(shardsToAllocate, node1Recoveries + node2Recoveries + node3Recoveries); + assertTrue(rerouteScheduled.get()); } - public void testAllocateAllPrimaryShardsAndPartialReplicaShardsUntilTimedOut() { + public void testAllocateAllPrimaryShardsAndPartialReplicaShardsUntilTimedOutAndRerouteScheduled() { int numberOfIndices = 2; int numberOfShards = 5; int numberOfReplicas = 1; @@ -158,11 +220,7 @@ public void testAllocateAllPrimaryShardsAndPartialReplicaShardsUntilTimedOut() { BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings.build(), new CountDownLatch(shardsToAllocate)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); RoutingAllocation allocation = new RoutingAllocation( yesAllocationDeciders(), new RoutingNodes(state, false), @@ -171,6 +229,18 @@ public void testAllocateAllPrimaryShardsAndPartialReplicaShardsUntilTimedOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List initializingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING); int node1Recoveries = allocation.routingNodes().getInitialPrimariesIncomingRecoveries(node1.getId()); @@ -179,20 +249,17 @@ public void testAllocateAllPrimaryShardsAndPartialReplicaShardsUntilTimedOut() { assertEquals(shardsToAllocate, initializingShards.size()); assertEquals(totalShardCount - shardsToAllocate, allocation.routingNodes().unassigned().ignored().size()); assertEquals(numberOfShards * numberOfIndices, node1Recoveries + node2Recoveries + node3Recoveries); + assertTrue(rerouteScheduled.get()); } - public void testAllShardsMoveWhenExcludedAndTimeoutNotBreached() { + public void testAllShardsMoveWhenExcludedAndTimeoutNotBreachedAndRerouteNotScheduled() { int numberOfIndices = 3; int numberOfShards = 5; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService(); state = applyStartedShardsUntilNoChange(state, allocationService); // check all shards allocated @@ -200,8 +267,7 @@ public void testAllShardsMoveWhenExcludedAndTimeoutNotBreached() { assertEquals(totalShardCount, state.getRoutingNodes().shardsWithState(STARTED).size()); int node1ShardCount = state.getRoutingNodes().node("node1").size(); Settings settings = Settings.builder().put("cluster.routing.allocation.exclude.zone", "1a").build(); - int shardsToMove = 10 + 1000; // such that time out is never breached - BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings, new CountDownLatch(shardsToMove)); + BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings, new CountDownLatch(Integer.MAX_VALUE)); RoutingAllocation allocation = new RoutingAllocation( allocationDecidersForExcludeAPI(settings), new RoutingNodes(state, false), @@ -210,30 +276,39 @@ public void testAllShardsMoveWhenExcludedAndTimeoutNotBreached() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(node1ShardCount, relocatingShards.size()); + assertFalse(rerouteScheduled.get()); } - public void testNoShardsMoveWhenExcludedAndTimeoutBreached() { + public void testNoShardsMoveWhenExcludedAndTimeoutBreachedAndRerouteScheduled() { int numberOfIndices = 3; int numberOfShards = 5; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService(); state = applyStartedShardsUntilNoChange(state, allocationService); // check all shards allocated assertEquals(0, state.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals(totalShardCount, state.getRoutingNodes().shardsWithState(STARTED).size()); Settings settings = Settings.builder().put("cluster.routing.allocation.exclude.zone", "1a").build(); - int shardsToMove = 0; // such that time out is never breached + int shardsToMove = 0; // such that time out is breached BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(settings, new CountDownLatch(shardsToMove)); RoutingAllocation allocation = new RoutingAllocation( allocationDecidersForExcludeAPI(settings), @@ -243,23 +318,32 @@ public void testNoShardsMoveWhenExcludedAndTimeoutBreached() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(0, relocatingShards.size()); + assertTrue(rerouteScheduled.get()); } - public void testPartialShardsMoveWhenExcludedAndTimeoutBreached() { + public void testPartialShardsMoveWhenExcludedAndTimeoutBreachedAndRerouteScheduled() { int numberOfIndices = 3; int numberOfShards = 5; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService(); state = applyStartedShardsUntilNoChange(state, allocationService); // check all shards allocated @@ -279,23 +363,32 @@ public void testPartialShardsMoveWhenExcludedAndTimeoutBreached() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(shardsToMove / 3, relocatingShards.size()); + assertTrue(rerouteScheduled.get()); } - public void testClusterRebalancedWhenNotTimedOut() { + public void testClusterRebalancedWhenNotTimedOutAndRerouteNotScheduled() { int numberOfIndices = 1; int numberOfShards = 15; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService( Settings.builder().put("cluster.routing.allocation.exclude.zone", "1a").build() ); // such that no shards are allocated to node1 @@ -306,8 +399,7 @@ public void testClusterRebalancedWhenNotTimedOut() { assertEquals(totalShardCount, state.getRoutingNodes().shardsWithState(STARTED).size()); assertEquals(0, node1ShardCount); Settings newSettings = Settings.builder().put("cluster.routing.allocation.exclude.zone", "").build(); - int shardsToMove = 1000; // such that time out is never breached - BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(newSettings, new CountDownLatch(shardsToMove)); + BalancedShardsAllocator allocator = new TestBalancedShardsAllocator(newSettings, new CountDownLatch(Integer.MAX_VALUE)); RoutingAllocation allocation = new RoutingAllocation( allocationDecidersForExcludeAPI(newSettings), new RoutingNodes(state, false), @@ -316,23 +408,32 @@ public void testClusterRebalancedWhenNotTimedOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(totalShardCount / 3, relocatingShards.size()); + assertFalse(rerouteScheduled.get()); } - public void testClusterNotRebalancedWhenTimedOut() { + public void testClusterNotRebalancedWhenTimedOutAndRerouteScheduled() { int numberOfIndices = 1; int numberOfShards = 15; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService( Settings.builder().put("cluster.routing.allocation.exclude.zone", "1a").build() ); // such that no shards are allocated to node1 @@ -353,23 +454,32 @@ public void testClusterNotRebalancedWhenTimedOut() { null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(0, relocatingShards.size()); + assertTrue(rerouteScheduled.get()); } - public void testClusterPartialRebalancedWhenTimedOut() { + public void testClusterPartialRebalancedWhenTimedOutAndRerouteScheduled() { int numberOfIndices = 1; int numberOfShards = 15; int numberOfReplicas = 1; int totalShardCount = numberOfIndices * (numberOfShards * (numberOfReplicas + 1)); Metadata metadata = buildMetadata(Metadata.builder(), numberOfIndices, numberOfShards, numberOfReplicas); RoutingTable routingTable = buildRoutingTable(metadata); - ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) - .metadata(metadata) - .routingTable(routingTable) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)) - .build(); + setupStateAndService(metadata, routingTable); MockAllocationService allocationService = createAllocationService( Settings.builder().put("cluster.routing.allocation.exclude.zone", "1a").build() ); // such that no shards are allocated to node1 @@ -404,9 +514,22 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca null, System.nanoTime() ); + AtomicBoolean rerouteScheduled = new AtomicBoolean(false); + final RerouteService rerouteService = (reason, priority, listener) -> { + if (randomBoolean()) { + listener.onFailure(new OpenSearchException("simulated")); + } else { + listener.onResponse(clusterService.state()); + } + assertEquals("reroute after balanced shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteScheduled.compareAndSet(false, true); + }; + allocator.setRerouteService(rerouteService); allocator.allocate(allocation); List relocatingShards = allocation.routingNodes().shardsWithState(ShardRoutingState.RELOCATING); assertEquals(3, relocatingShards.size()); + assertTrue(rerouteScheduled.get()); } public void testAllocatorNeverTimedOutIfValueIsMinusOne() { diff --git a/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java index c7eae77d6deba..ebc2e59fa5a30 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.ClusterInfo; @@ -22,6 +23,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.RoutingNodes; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; @@ -30,6 +32,8 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -37,7 +41,9 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.core.index.shard.ShardId; import org.opensearch.snapshots.SnapshotShardSizeInfo; +import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.gateway.TestShardBatchGatewayAllocator; +import org.opensearch.threadpool.TestThreadPool; import org.junit.Before; import java.util.ArrayList; @@ -47,13 +53,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.opensearch.gateway.ShardsBatchGatewayAllocator.PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING; import static org.opensearch.gateway.ShardsBatchGatewayAllocator.PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING_KEY; import static org.opensearch.gateway.ShardsBatchGatewayAllocator.REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING; import static org.opensearch.gateway.ShardsBatchGatewayAllocator.REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING_KEY; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class GatewayAllocatorTests extends OpenSearchAllocationTestCase { @@ -426,22 +432,62 @@ public void testReplicaAllocatorTimeout() { assertEquals(-1, REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(build).getMillis()); } - public void testCollectTimedOutShards() throws InterruptedException { + public void testCollectTimedOutShardsAndScheduleReroute_Success() throws InterruptedException { createIndexAndUpdateClusterState(2, 5, 2); - CountDownLatch latch = new CountDownLatch(10); - testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator(latch); + TestThreadPool threadPool = new TestThreadPool(getTestName()); + ClusterService clusterService = ClusterServiceUtils.createClusterService(clusterState, threadPool); + final CountDownLatch rerouteLatch = new CountDownLatch(2); + final RerouteService rerouteService = (reason, priority, listener) -> { + listener.onResponse(clusterService.state()); + assertThat(rerouteLatch.getCount(), greaterThanOrEqualTo(0L)); + assertEquals("reroute after existing shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteLatch.countDown(); + }; + CountDownLatch timedOutShardsLatch = new CountDownLatch(20); + testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator(timedOutShardsLatch, 1000, rerouteService); testShardsBatchGatewayAllocator.setPrimaryBatchAllocatorTimeout(TimeValue.ZERO); testShardsBatchGatewayAllocator.setReplicaBatchAllocatorTimeout(TimeValue.ZERO); BatchRunnableExecutor executor = testShardsBatchGatewayAllocator.allocateAllUnassignedShards(testAllocation, true); executor.run(); - assertTrue(latch.await(1, TimeUnit.MINUTES)); - latch = new CountDownLatch(10); - testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator(latch); + assertEquals(timedOutShardsLatch.getCount(), 10); + assertEquals(1, rerouteLatch.getCount()); + executor = testShardsBatchGatewayAllocator.allocateAllUnassignedShards(testAllocation, false); + executor.run(); + assertEquals(timedOutShardsLatch.getCount(), 0); + assertEquals(0, rerouteLatch.getCount()); // even with failure it doesn't leak any listeners + final boolean terminated = terminate(threadPool); + assert terminated; + clusterService.close(); + } + + public void testCollectTimedOutShardsAndScheduleReroute_Failure() throws InterruptedException { + createIndexAndUpdateClusterState(2, 5, 2); + TestThreadPool threadPool = new TestThreadPool(getTestName()); + ClusterService clusterService = ClusterServiceUtils.createClusterService(clusterState, threadPool); + final CountDownLatch rerouteLatch = new CountDownLatch(2); + final RerouteService rerouteService = (reason, priority, listener) -> { + listener.onFailure(new OpenSearchException("simulated")); + assertThat(rerouteLatch.getCount(), greaterThanOrEqualTo(0L)); + assertEquals("reroute after existing shards allocator timed out", reason); + assertEquals(Priority.HIGH, priority); + rerouteLatch.countDown(); + }; + CountDownLatch timedOutShardsLatch = new CountDownLatch(20); + testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator(timedOutShardsLatch, 1000, rerouteService); testShardsBatchGatewayAllocator.setPrimaryBatchAllocatorTimeout(TimeValue.ZERO); testShardsBatchGatewayAllocator.setReplicaBatchAllocatorTimeout(TimeValue.ZERO); + BatchRunnableExecutor executor = testShardsBatchGatewayAllocator.allocateAllUnassignedShards(testAllocation, true); + executor.run(); + assertEquals(timedOutShardsLatch.getCount(), 10); + assertEquals(1, rerouteLatch.getCount()); executor = testShardsBatchGatewayAllocator.allocateAllUnassignedShards(testAllocation, false); executor.run(); - assertTrue(latch.await(1, TimeUnit.MINUTES)); + assertEquals(timedOutShardsLatch.getCount(), 0); + assertEquals(0, rerouteLatch.getCount()); // even with failure it doesn't leak any listeners + final boolean terminated = terminate(threadPool); + assert terminated; + clusterService.close(); } private void createIndexAndUpdateClusterState(int count, int numberOfShards, int numberOfReplicas) { diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java index 156b1d7c620e6..c2ff228a6bf3a 100644 --- a/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java +++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java @@ -10,6 +10,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision; import org.opensearch.cluster.routing.allocation.RoutingAllocation; @@ -39,12 +40,13 @@ public TestShardBatchGatewayAllocator() { } - public TestShardBatchGatewayAllocator(CountDownLatch latch) { + public TestShardBatchGatewayAllocator(CountDownLatch latch, long maxBatchSize, RerouteService rerouteService) { + super(maxBatchSize, rerouteService); this.latch = latch; } public TestShardBatchGatewayAllocator(long maxBatchSize) { - super(maxBatchSize); + super(maxBatchSize, null); } Map> knownAllocations = new HashMap<>(); From 19356506f3eeae30031536288793662caf05bf65 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:56:05 +0530 Subject: [PATCH 060/357] Add dedicated string prefix for remote index metadata and remote routing table (#15575) * Hashed prefix for index metadata Signed-off-by: Sooraj Sinha --- .../remote/RemoteStatePublicationIT.java | 49 +++++++++- .../RemoteStoreClusterStateRestoreIT.java | 13 ++- .../remote/RemoteWriteableBlobEntity.java | 11 +++ .../RemoteWriteableEntityBlobStore.java | 2 +- .../common/settings/ClusterSettings.java | 5 + .../remote/RemoteClusterStateService.java | 17 +++- .../remote/RemoteIndexMetadataManager.java | 48 ++++++++++ .../remote/model/RemoteIndexMetadata.java | 30 +++++- .../model/RemoteRoutingTableBlobStore.java | 14 +++ .../RemoteIndexMetadataManagerTests.java | 35 ++++++- .../model/RemoteIndexMetadataTests.java | 91 +++++++++++++++++-- 11 files changed, 298 insertions(+), 17 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 0e6321867a33b..f794eec65cb1b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -17,7 +17,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.discovery.DiscoveryStats; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore; +import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; +import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.repositories.RepositoriesService; @@ -48,19 +52,27 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase { - private static String INDEX_NAME = "test-index"; + private static final String INDEX_NAME = "test-index"; + private static final String REMOTE_STATE_PREFIX = "!"; + private static final String REMOTE_ROUTING_PREFIX = "_"; private boolean isRemoteStateEnabled = true; private String isRemotePublicationEnabled = "true"; + private boolean hasRemoteStateCharPrefix; + private boolean hasRemoteRoutingCharPrefix; @Before public void setup() { asyncUploadMockFsRepo = false; isRemoteStateEnabled = true; isRemotePublicationEnabled = "true"; + hasRemoteStateCharPrefix = randomBoolean(); + hasRemoteRoutingCharPrefix = randomBoolean(); } @Override @@ -84,6 +96,7 @@ protected Settings nodeSettings(int nodeOrdinal) { "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, routingTableRepoName ); + return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled) @@ -94,6 +107,19 @@ protected Settings nodeSettings(int nodeOrdinal) { RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE ) + .put( + RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.getKey(), + hasRemoteStateCharPrefix ? REMOTE_STATE_PREFIX : "" + ) + .put( + RemoteRoutingTableBlobStore.CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX.getKey(), + hasRemoteRoutingCharPrefix ? REMOTE_ROUTING_PREFIX : "" + ) + .put(RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX.toString()) + .put( + RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING.getKey(), + PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString() + ) .build(); } @@ -137,6 +163,27 @@ public void testPublication() throws Exception { Map manifestFiles = getMetadataFiles(repository, RemoteClusterMetadataManifest.MANIFEST); assertTrue(manifestFiles.containsKey(RemoteClusterMetadataManifest.MANIFEST)); + RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance( + RemoteClusterStateService.class, + internalCluster().getClusterManagerName() + ); + ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().metadata().clusterUUID() + ).get(); + assertThat(manifest.getIndices().size(), is(1)); + if (hasRemoteStateCharPrefix) { + for (UploadedIndexMetadata md : manifest.getIndices()) { + assertThat(md.getUploadedFilename(), startsWith(REMOTE_STATE_PREFIX)); + } + } + assertThat(manifest.getIndicesRouting().size(), is(1)); + if (hasRemoteRoutingCharPrefix) { + for (UploadedIndexMetadata md : manifest.getIndicesRouting()) { + assertThat(md.getUploadedFilename(), startsWith(REMOTE_ROUTING_PREFIX)); + } + } + // get settings from each node and verify that it is updated Settings settings = clusterService().getSettings(); logger.info("settings : {}", settings); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index 6ec973090883b..d078ba05faa12 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -328,10 +328,17 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh internalCluster().stopAllNodes(); // Step - 3 Delete index metadata file in remote try { - Files.move( - segmentRepoPath.resolve(encodeString(clusterName) + "/cluster-state/" + prevClusterUUID + "/index"), - segmentRepoPath.resolve("cluster-state/") + RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance( + RemoteClusterStateService.class, + internalCluster().getClusterManagerName() ); + ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().metadata().clusterUUID() + ).get(); + for (UploadedIndexMetadata md : manifest.getIndices()) { + Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/")); + } } catch (IOException e) { throw new RuntimeException(e); } diff --git a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableBlobEntity.java b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableBlobEntity.java index f034ce2d1adf1..4d9adb48d2a3a 100644 --- a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableBlobEntity.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableBlobEntity.java @@ -63,6 +63,17 @@ public String[] getBlobPathTokens() { public abstract String generateBlobFileName(); + /** + * Generate the blob path for the remote entity by adding a custom prefix. + * This custom prefix may be generated by any of the strategies defined in {@link org.opensearch.index.remote.RemoteStoreEnums} + * The default implementation returns the same path as passed in the argument. + * @param blobPath The remote path on which the remote entity is to be uploaded + * @return The modified remote path after adding a custom prefix at which the remote entity will be uploaded. + */ + public BlobPath getPrefixedPath(BlobPath blobPath) { + return blobPath; + } + public String clusterUUID() { return clusterUUID; } diff --git a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntityBlobStore.java b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntityBlobStore.java index baa44a7c9bde9..b5e074874dd38 100644 --- a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntityBlobStore.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntityBlobStore.java @@ -102,7 +102,7 @@ public BlobPath getBlobPathForUpload(final RemoteWriteableBlobEntity obj) { for (String token : obj.getBlobPathParameters().getPathTokens()) { blobPath = blobPath.add(token); } - return blobPath; + return obj.getPrefixedPath(blobPath); } public BlobPath getBlobPathForDownload(final RemoteWriteableBlobEntity obj) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 136eeb4e33251..c6a3ee6df609b 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -108,6 +108,7 @@ import org.opensearch.gateway.ShardsBatchGatewayAllocator; import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.RemoteIndexMetadataManager; import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore; import org.opensearch.http.HttpTransportSettings; import org.opensearch.index.IndexModule; @@ -736,6 +737,9 @@ public void apply(Settings value, Settings current, Settings previous) { GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.REMOTE_STATE_READ_TIMEOUT_SETTING, + RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX, + RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_TYPE_SETTING, + RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, @@ -743,6 +747,7 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING, + RemoteRoutingTableBlobStore.CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX, // Admission Control Settings AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index a223bfbe736c3..fe34b68702c41 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -150,6 +150,16 @@ public class RemoteClusterStateService implements Closeable { Setting.Property.NodeScope ); + /** + * Controls the fixed prefix for the cluster state path on remote store. + */ + public static final Setting CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX = Setting.simpleString( + "cluster.remote_store.state.path.prefix", + "", + Property.NodeScope, + Property.Final + ); + /** * Validation mode for cluster state checksum. * None: Validation will be disabled. @@ -211,6 +221,7 @@ public static RemoteClusterStateValidationMode parseString(String mode) { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]"; private final boolean isPublicationEnabled; + private final String remotePathPrefix; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -252,6 +263,7 @@ public RemoteClusterStateService( this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings) && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings); + this.remotePathPrefix = CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.get(settings); this.remoteRoutingTableService = RemoteRoutingTableServiceFactory.getService( repositoriesService, settings, @@ -728,7 +740,10 @@ UploadedMetadataResults writeMetadataInParallel( indexMetadata, clusterState.metadata().clusterUUID(), blobStoreRepository.getCompressor(), - blobStoreRepository.getNamedXContentRegistry() + blobStoreRepository.getNamedXContentRegistry(), + remoteIndexMetadataManager.getPathTypeSetting(), + remoteIndexMetadataManager.getPathHashAlgoSetting(), + remotePathPrefix ), listener ); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index d1f08a7c2a33d..d8e8ffc68834d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -20,6 +20,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.model.RemoteIndexMetadata; import org.opensearch.gateway.remote.model.RemoteReadResult; +import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -44,11 +45,38 @@ public class RemoteIndexMetadataManager extends AbstractRemoteWritableEntityMana Setting.Property.Deprecated ); + /** + * This setting is used to set the remote index metadata blob store path type strategy. + */ + public static final Setting REMOTE_INDEX_METADATA_PATH_TYPE_SETTING = new Setting<>( + "cluster.remote_store.index_metadata.path_type", + RemoteStoreEnums.PathType.HASHED_PREFIX.toString(), + RemoteStoreEnums.PathType::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * This setting is used to set the remote index metadata blob store path hash algorithm strategy. + * This setting will come to effect if the {@link #REMOTE_INDEX_METADATA_PATH_TYPE_SETTING} + * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}. + */ + public static final Setting REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING = new Setting<>( + "cluster.remote_store.index_metadata.path_hash_algo", + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(), + RemoteStoreEnums.PathHashAlgorithm::parseString, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private final Compressor compressor; private final NamedXContentRegistry namedXContentRegistry; private volatile TimeValue indexMetadataUploadTimeout; + private RemoteStoreEnums.PathType pathType; + private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; + public RemoteIndexMetadataManager( ClusterSettings clusterSettings, String clusterName, @@ -70,7 +98,11 @@ public RemoteIndexMetadataManager( this.namedXContentRegistry = blobStoreRepository.getNamedXContentRegistry(); this.compressor = blobStoreRepository.getCompressor(); this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); + this.pathType = clusterSettings.get(REMOTE_INDEX_METADATA_PATH_TYPE_SETTING); + this.pathHashAlgo = clusterSettings.get(REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING); clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); + clusterSettings.addSettingsUpdateConsumer(REMOTE_INDEX_METADATA_PATH_TYPE_SETTING, this::setPathTypeSetting); + clusterSettings.addSettingsUpdateConsumer(REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting); } /** @@ -127,4 +159,20 @@ protected ActionListener getWrappedReadListener( ex -> listener.onFailure(new RemoteStateTransferException("Download failed for " + component, remoteEntity, ex)) ); } + + private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) { + this.pathType = pathType; + } + + private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) { + this.pathHashAlgo = pathHashAlgo; + } + + protected RemoteStoreEnums.PathType getPathTypeSetting() { + return pathType; + } + + protected RemoteStoreEnums.PathHashAlgorithm getPathHashAlgoSetting() { + return pathHashAlgo; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java index 5308f92c633b1..97a115a44484c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java @@ -9,6 +9,7 @@ package org.opensearch.gateway.remote.model; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.Streams; import org.opensearch.common.remote.AbstractClusterMetadataWriteableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; @@ -17,6 +18,8 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; @@ -41,15 +44,24 @@ public class RemoteIndexMetadata extends AbstractClusterMetadataWriteableBlobEnt public static final String INDEX = "index"; private IndexMetadata indexMetadata; + private RemoteStoreEnums.PathType pathType; + private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; + private String fixedPrefix; public RemoteIndexMetadata( final IndexMetadata indexMetadata, final String clusterUUID, final Compressor compressor, - final NamedXContentRegistry namedXContentRegistry + final NamedXContentRegistry namedXContentRegistry, + final RemoteStoreEnums.PathType pathType, + final RemoteStoreEnums.PathHashAlgorithm pathHashAlgo, + final String fixedPrefix ) { super(clusterUUID, compressor, namedXContentRegistry); this.indexMetadata = indexMetadata; + this.pathType = pathType; + this.pathHashAlgo = pathHashAlgo; + this.fixedPrefix = fixedPrefix; } public RemoteIndexMetadata( @@ -86,6 +98,22 @@ public String generateBlobFileName() { return blobFileName; } + @Override + public BlobPath getPrefixedPath(BlobPath blobPath) { + if (pathType == null) { + return blobPath; + } + assert pathHashAlgo != null; + return pathType.path( + RemoteStorePathStrategy.PathInput.builder() + .fixedPrefix(fixedPrefix) + .basePath(blobPath) + .indexUUID(indexMetadata.getIndexUUID()) + .build(), + pathHashAlgo + ); + } + @Override public UploadedMetadata getUploadedMetadata() { assert blobName != null; diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java index 1a28c97dc8a77..8490f92d055eb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteRoutingTableBlobStore.java @@ -15,6 +15,7 @@ import org.opensearch.common.remote.RemoteWriteableEntityBlobStore; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; import org.opensearch.index.remote.RemoteStoreEnums; @@ -58,8 +59,19 @@ public class RemoteRoutingTableBlobStore CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX = Setting.simpleString( + "cluster.remote_store.routing_table.path.prefix", + "", + Property.NodeScope, + Property.Final + ); + private RemoteStoreEnums.PathType pathType; private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo; + private String pathPrefix; public RemoteRoutingTableBlobStore( BlobStoreTransferService blobStoreTransferService, @@ -79,6 +91,7 @@ public RemoteRoutingTableBlobStore( ); this.pathType = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING); this.pathHashAlgo = clusterSettings.get(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING); + this.pathPrefix = clusterSettings.get(CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX); clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING, this::setPathTypeSetting); clusterSettings.addSettingsUpdateConsumer(REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting); } @@ -90,6 +103,7 @@ public BlobPath getBlobPathForUpload(final RemoteWriteableBlobEntity(listener, latch) ); latch.await(); @@ -132,7 +136,7 @@ public void testGetAsyncWriteRunnable_IOFailure() throws Exception { remoteIndexMetadataManager.writeAsync( INDEX, - new RemoteIndexMetadata(indexMetadata, "cluster-uuid", compressor, null), + new RemoteIndexMetadata(indexMetadata, "cluster-uuid", compressor, null, null, null, null), new LatchedActionListener<>(listener, latch) ); latch.await(); @@ -182,6 +186,31 @@ public void testGetAsyncReadRunnable_IOFailure() throws Exception { assertTrue(listener.getFailure() instanceof RemoteStateTransferException); } + public void testRemoteIndexMetadataPathTypeSetting() { + // Assert the default is HASHED_PREFIX + assertEquals(HASHED_PREFIX.toString(), remoteIndexMetadataManager.getPathTypeSetting().toString()); + + Settings newSettings = Settings.builder() + .put("cluster.remote_store.index_metadata.path_type", RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(RemoteStoreEnums.PathType.FIXED.toString(), remoteIndexMetadataManager.getPathTypeSetting().toString()); + } + + public void testRemoteIndexMetadataHashAlgoSetting() { + // Assert the default is FNV_1A_BASE64 + assertEquals(FNV_1A_BASE64.toString(), remoteIndexMetadataManager.getPathHashAlgoSetting().toString()); + + Settings newSettings = Settings.builder() + .put("cluster.remote_store.index_metadata.path_hash_algo", RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString()) + .build(); + clusterSettings.applySettings(newSettings); + assertEquals( + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(), + remoteIndexMetadataManager.getPathHashAlgoSetting().toString() + ); + } + private IndexMetadata getIndexMetadata(String name, @Nullable Boolean writeIndex, String... aliases) { IndexMetadata.Builder builder = IndexMetadata.builder(name) .settings( diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java index 7f9c3fdbae91b..9923f23875af0 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java @@ -23,6 +23,8 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; +import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.IndicesModule; @@ -93,7 +95,15 @@ public void tearDown() throws Exception { public void testClusterUUID() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata( @@ -107,7 +117,15 @@ public void testClusterUUID() { public void testFullBlobName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata( @@ -121,7 +139,15 @@ public void testFullBlobName() { public void testBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata( @@ -135,7 +161,15 @@ public void testBlobFileName() { public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(INDEX, indexMetadata.getIndexUUID()))); assertThat(params.getFilePrefix(), is("metadata")); @@ -143,7 +177,15 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertThat(nameTokens[0], is("metadata")); @@ -154,7 +196,15 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); @@ -163,7 +213,15 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + null, + null, + null + ); try (InputStream inputStream = remoteObjectForUpload.serialize()) { assertThat(inputStream.available(), greaterThan(0)); IndexMetadata readIndexMetadata = remoteObjectForUpload.deserialize(inputStream); @@ -171,6 +229,25 @@ public void testSerDe() throws IOException { } } + public void testPrefixedPath() { + IndexMetadata indexMetadata = getIndexMetadata(); + String fixedPrefix = "*"; + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata( + indexMetadata, + clusterUUID, + compressor, + namedXContentRegistry, + PathType.HASHED_PREFIX, + PathHashAlgorithm.FNV_1A_COMPOSITE_1, + fixedPrefix + ); + String testPath = "test-path"; + String expectedPath = fixedPrefix + "410100110100101/test-path/index-uuid/"; + BlobPath prefixedPath = remoteObjectForUpload.getPrefixedPath(BlobPath.cleanPath().add(testPath)); + assertThat(prefixedPath.buildAsString(), is(expectedPath)); + + } + private IndexMetadata getIndexMetadata() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() From b957eca9306dc8b96fe0bc675404b95ad68d262f Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 5 Sep 2024 11:02:24 -0400 Subject: [PATCH 061/357] Fix MacOS assemble workflows (#15741) Signed-off-by: Andriy Redko --- .github/workflows/assemble.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index 200d9cf2d788b..294627622a136 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -30,11 +30,8 @@ jobs: - name: Setup docker (missing on MacOS) id: setup_docker if: runner.os == 'macos' - uses: douglascamata/setup-docker-macos-action@main - continue-on-error: true - with: - upgrade-qemu: true - colima: v0.6.8 + run: | + exit 0; - name: Run Gradle (assemble) if: runner.os == 'macos' && steps.setup_docker.outcome != 'success' run: | @@ -48,4 +45,4 @@ jobs: - name: Run Gradle (assemble) if: runner.os == 'macos' && steps.setup_docker.outcome == 'success' run: | - ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} + exit 0; From 2f1e2098d2015ccc1e795df5b05b0236202b8ee7 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 5 Sep 2024 12:12:13 -0400 Subject: [PATCH 062/357] Add 2.18.0 version (#15712) Signed-off-by: Andriy Redko --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 980e08c696d54..0f59fec3d824a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -38,3 +38,4 @@ BWC_VERSION: - "2.16.0" - "2.16.1" - "2.17.0" + - "2.18.0" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 28cb989185ff7..ca5dd306cf907 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -109,6 +109,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From c505709d73587fd2fdb714d24510b44ae07aa4a6 Mon Sep 17 00:00:00 2001 From: Ganesh Krishna Ramadurai Date: Thu, 5 Sep 2024 10:16:53 -0700 Subject: [PATCH 063/357] Provide factory for pluggable deciders (#15713) Signed-off-by: Ganesh Ramadurai --- .../main/java/org/opensearch/node/Node.java | 8 +-- .../org/opensearch/plugins/SearchPlugin.java | 10 +-- .../search/DefaultSearchContext.java | 32 +++++---- .../org/opensearch/search/SearchModule.java | 24 +++---- .../org/opensearch/search/SearchService.java | 10 +-- .../deciders/ConcurrentSearchDecision.java | 2 +- ...va => ConcurrentSearchRequestDecider.java} | 36 ++++++---- .../deciders/ConcurrentSearchVisitor.java | 6 +- .../search/DefaultSearchContextTests.java | 70 +++++++++++-------- .../opensearch/search/SearchModuleTests.java | 49 ++++++------- .../java/org/opensearch/node/MockNode.java | 6 +- 11 files changed, 136 insertions(+), 117 deletions(-) rename server/src/main/java/org/opensearch/search/deciders/{ConcurrentSearchDecider.java => ConcurrentSearchRequestDecider.java} (50%) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 34ed957542aff..a8d4ebcf23dab 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -235,7 +235,7 @@ import org.opensearch.search.aggregations.support.AggregationUsageService; import org.opensearch.search.backpressure.SearchBackpressureService; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.query.QueryPhase; @@ -1344,7 +1344,7 @@ protected Node( circuitBreakerService, searchModule.getIndexSearcherExecutor(threadPool), taskResourceTrackingService, - searchModule.getConcurrentSearchDeciders() + searchModule.getConcurrentSearchRequestDeciderFactories() ); final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) @@ -2004,7 +2004,7 @@ protected SearchService newSearchService( CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDecidersList + Collection concurrentSearchDeciderFactories ) { return new SearchService( clusterService, @@ -2018,7 +2018,7 @@ protected SearchService newSearchService( circuitBreakerService, indexSearcherExecutor, taskResourceTrackingService, - concurrentSearchDecidersList + concurrentSearchDeciderFactories ); } diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index 498da4042fa33..710dd32371f83 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -65,7 +65,7 @@ import org.opensearch.search.aggregations.pipeline.MovAvgPipelineAggregator; import org.opensearch.search.aggregations.pipeline.PipelineAggregator; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.highlight.Highlighter; import org.opensearch.search.query.QueryPhaseSearcher; @@ -141,12 +141,12 @@ default Map getHighlighters() { } /** - * Allows plugins to register custom decider for concurrent search - * @return A {@link ConcurrentSearchDecider} + * Allows plugins to register a factory to create custom decider for concurrent search + * @return A {@link ConcurrentSearchRequestDecider.Factory} */ @ExperimentalApi - default ConcurrentSearchDecider getConcurrentSearchDecider() { - return null; + default Optional getConcurrentSearchRequestDeciderFactory() { + return Optional.empty(); } /** diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java index 1706c27ccf922..74a7482d975df 100644 --- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java @@ -72,8 +72,8 @@ import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.collapse.CollapseContext; -import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.deciders.ConcurrentSearchDecision; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.deciders.ConcurrentSearchVisitor; import org.opensearch.search.dfs.DfsSearchResult; import org.opensearch.search.fetch.FetchPhase; @@ -106,13 +106,14 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.LongSupplier; -import java.util.stream.Collectors; import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_MODE; @@ -137,7 +138,7 @@ final class DefaultSearchContext extends SearchContext { private final ShardSearchRequest request; private final SearchShardTarget shardTarget; private final LongSupplier relativeTimeSupplier; - private final Collection concurrentSearchDeciders; + private final Collection concurrentSearchDeciderFactories; private SearchType searchType; private final BigArrays bigArrays; private final IndexShard indexShard; @@ -223,7 +224,7 @@ final class DefaultSearchContext extends SearchContext { boolean validate, Executor executor, Function requestToAggReduceContextBuilder, - Collection concurrentSearchDeciders + Collection concurrentSearchDeciderFactories ) throws IOException { this.readerContext = readerContext; this.request = request; @@ -267,7 +268,7 @@ final class DefaultSearchContext extends SearchContext { this.maxAggRewriteFilters = evaluateFilterRewriteSetting(); this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold(); - this.concurrentSearchDeciders = concurrentSearchDeciders; + this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories; this.keywordIndexOrDocValuesEnabled = evaluateKeywordIndexOrDocValuesEnabled(); } @@ -932,14 +933,21 @@ public boolean shouldUseConcurrentSearch() { private boolean evaluateAutoMode() { - // filter out deciders that want to opt-out of decision-making - final Set filteredDeciders = concurrentSearchDeciders.stream() - .filter(concurrentSearchDecider -> concurrentSearchDecider.canEvaluateForIndex(indexService.getIndexSettings())) - .collect(Collectors.toSet()); + final Set concurrentSearchRequestDeciders = new HashSet<>(); + + // create the ConcurrentSearchRequestDeciders using registered factories + for (ConcurrentSearchRequestDecider.Factory deciderFactory : concurrentSearchDeciderFactories) { + final Optional concurrentSearchRequestDecider = deciderFactory.create( + indexService.getIndexSettings() + ); + concurrentSearchRequestDecider.ifPresent(concurrentSearchRequestDeciders::add); + + } + // evaluate based on concurrent search query visitor - if (filteredDeciders.size() > 0) { + if (concurrentSearchRequestDeciders.size() > 0) { ConcurrentSearchVisitor concurrentSearchVisitor = new ConcurrentSearchVisitor( - filteredDeciders, + concurrentSearchRequestDeciders, indexService.getIndexSettings() ); if (request().source() != null && request().source().query() != null) { @@ -949,7 +957,7 @@ private boolean evaluateAutoMode() { } final List decisions = new ArrayList<>(); - for (ConcurrentSearchDecider decider : filteredDeciders) { + for (ConcurrentSearchRequestDecider decider : concurrentSearchRequestDeciders) { ConcurrentSearchDecision decision = decider.getConcurrentSearchDecision(); if (decision != null) { if (logger.isDebugEnabled()) { diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index c51004a1ea95e..3a746259af3b5 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -239,7 +239,7 @@ import org.opensearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; import org.opensearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.ExplainPhase; @@ -318,7 +318,7 @@ public class SearchModule { private final QueryPhaseSearcher queryPhaseSearcher; private final SearchPlugin.ExecutorServiceProvider indexSearcherExecutorProvider; - private final Collection concurrentSearchDeciders; + private final Collection concurrentSearchDeciderFactories; /** * Constructs a new SearchModule object @@ -348,25 +348,23 @@ public SearchModule(Settings settings, List plugins) { queryPhaseSearcher = registerQueryPhaseSearcher(plugins); indexSearcherExecutorProvider = registerIndexSearcherExecutorProvider(plugins); namedWriteables.addAll(SortValue.namedWriteables()); - concurrentSearchDeciders = registerConcurrentSearchDeciders(plugins); + concurrentSearchDeciderFactories = registerConcurrentSearchDeciderFactories(plugins); } - private Collection registerConcurrentSearchDeciders(List plugins) { - List concurrentSearchDeciders = new ArrayList<>(); + private Collection registerConcurrentSearchDeciderFactories(List plugins) { + List concurrentSearchDeciderFactories = new ArrayList<>(); for (SearchPlugin plugin : plugins) { - ConcurrentSearchDecider decider = plugin.getConcurrentSearchDecider(); - if (decider != null) { - concurrentSearchDeciders.add(decider); - } + final Optional deciderFactory = plugin.getConcurrentSearchRequestDeciderFactory(); + deciderFactory.ifPresent(concurrentSearchDeciderFactories::add); } - return concurrentSearchDeciders; + return concurrentSearchDeciderFactories; } /** - * Returns the concurrent search deciders that the plugins have registered + * Returns the concurrent search decider factories that the plugins have registered */ - public Collection getConcurrentSearchDeciders() { - return concurrentSearchDeciders; + public Collection getConcurrentSearchRequestDeciderFactories() { + return concurrentSearchDeciderFactories; } public List getNamedWriteables() { diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index e2a804a674d8f..40afdbfbdaa9e 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -104,7 +104,7 @@ import org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.collapse.CollapseContext; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.dfs.DfsPhase; import org.opensearch.search.dfs.DfsSearchResult; import org.opensearch.search.fetch.FetchPhase; @@ -364,7 +364,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final QueryPhase queryPhase; private final FetchPhase fetchPhase; - private final Collection concurrentSearchDeciders; + private final Collection concurrentSearchDeciderFactories; private volatile long defaultKeepAlive; @@ -410,7 +410,7 @@ public SearchService( CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDeciders + Collection concurrentSearchDeciderFactories ) { Settings settings = clusterService.getSettings(); this.threadPool = threadPool; @@ -466,7 +466,7 @@ public SearchService( allowDerivedField = CLUSTER_ALLOW_DERIVED_FIELD_SETTING.get(settings); clusterService.getClusterSettings().addSettingsUpdateConsumer(CLUSTER_ALLOW_DERIVED_FIELD_SETTING, this::setAllowDerivedField); - this.concurrentSearchDeciders = concurrentSearchDeciders; + this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories; } private void validateKeepAlives(TimeValue defaultKeepAlive, TimeValue maxKeepAlive) { @@ -1167,7 +1167,7 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear validate, indexSearcherExecutor, this::aggReduceContextBuilder, - concurrentSearchDeciders + concurrentSearchDeciderFactories ); // we clone the query shard context here just for rewriting otherwise we // might end up with incorrect state since we are using now() or script services diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java index 2a30413eff9c8..4ac47221856d1 100644 --- a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecision.java @@ -13,7 +13,7 @@ import java.util.Collection; /** - * This Class defines the decisions that a {@link ConcurrentSearchDecider#getConcurrentSearchDecision} can return. + * This Class defines the decisions that a {@link ConcurrentSearchRequestDecider#getConcurrentSearchDecision} can return. * */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchRequestDecider.java similarity index 50% rename from server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java rename to server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchRequestDecider.java index 9c588bb45b4ec..ec40527314454 100644 --- a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchDecider.java +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchRequestDecider.java @@ -12,17 +12,21 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilder; +import java.util.Optional; + /** - * {@link ConcurrentSearchDecider} allows pluggable way to evaluate if a query in the search request + * {@link ConcurrentSearchRequestDecider} allows pluggable way to evaluate if a query in the search request * can use concurrent segment search using the passed in queryBuilders from query tree and index settings * on a per shard request basis. - * Implementations can also opt out of the evaluation process for certain indices based on the index settings. - * For all the deciders which can evaluate query tree for an index, its evaluateForQuery method - * will be called for each node in the query tree. After traversing of the query tree is completed, the final - * decision from the deciders will be obtained using {@link ConcurrentSearchDecider#getConcurrentSearchDecision} + * Implementations will need to implement the Factory interface that can be used to create the ConcurrentSearchRequestDecider + * This factory will be called on each shard search request to create the ConcurrentSearchRequestDecider and get the + * concurrent search decision from the created decider on a per-request basis. + * For all the deciders the evaluateForQuery method will be called for each node in the query tree. + * After traversing of the query tree is completed, the final decision from the deciders will be + * obtained using {@link ConcurrentSearchRequestDecider#getConcurrentSearchDecision} */ @ExperimentalApi -public abstract class ConcurrentSearchDecider { +public abstract class ConcurrentSearchRequestDecider { /** * Evaluate for the passed in queryBuilder node in the query tree of the search request @@ -31,14 +35,6 @@ public abstract class ConcurrentSearchDecider { */ public abstract void evaluateForQuery(QueryBuilder queryBuilder, IndexSettings indexSettings); - /** - * Provides a way for deciders to opt out of decision-making process for certain requests based on - * index settings. - * Return true if interested in decision making for index, - * false, otherwise - */ - public abstract boolean canEvaluateForIndex(IndexSettings indexSettings); - /** * Provide the final decision for concurrent search based on all evaluations * Plugins may need to maintain internal state of evaluations to provide a final decision @@ -47,4 +43,16 @@ public abstract class ConcurrentSearchDecider { */ public abstract ConcurrentSearchDecision getConcurrentSearchDecision(); + /** + * Factory interface that can be implemented to create the ConcurrentSearchRequestDecider object. + * Implementations can use the passed in indexSettings to decide whether to create the decider object or + * return {@link Optional#empty()}. + */ + @ExperimentalApi + public interface Factory { + default Optional create(IndexSettings indexSettings) { + return Optional.empty(); + } + } + } diff --git a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java index 12ba1b2a9cc5f..d1a4fa982dc7e 100644 --- a/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java +++ b/server/src/main/java/org/opensearch/search/deciders/ConcurrentSearchVisitor.java @@ -19,15 +19,15 @@ /** * Class to traverse the QueryBuilder tree and invoke the - * {@link ConcurrentSearchDecider#evaluateForQuery} at each node of the query tree + * {@link ConcurrentSearchRequestDecider#evaluateForQuery} at each node of the query tree */ @ExperimentalApi public class ConcurrentSearchVisitor implements QueryBuilderVisitor { - private final Set deciders; + private final Set deciders; private final IndexSettings indexSettings; - public ConcurrentSearchVisitor(Set concurrentSearchVisitorDeciders, IndexSettings idxSettings) { + public ConcurrentSearchVisitor(Set concurrentSearchVisitorDeciders, IndexSettings idxSettings) { Objects.requireNonNull(concurrentSearchVisitorDeciders, "Concurrent search deciders cannot be null"); deciders = concurrentSearchVisitorDeciders; indexSettings = idxSettings; diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 7e213218eb97b..55b30d5068daa 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -76,8 +76,8 @@ import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.search.aggregations.SearchContextAggregations; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.deciders.ConcurrentSearchDecider; import org.opensearch.search.deciders.ConcurrentSearchDecision; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.LegacyReaderContext; import org.opensearch.search.internal.PitReaderContext; @@ -96,6 +96,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -984,14 +986,34 @@ protected Engine.Searcher acquireSearcherInternal(String source) { // Case4: multiple deciders are registered and all of them opt out of decision-making // with supported agg query so concurrent path is used - ConcurrentSearchDecider decider1 = mock(ConcurrentSearchDecider.class); - when(decider1.canEvaluateForIndex(any())).thenReturn(false); - ConcurrentSearchDecider decider2 = mock(ConcurrentSearchDecider.class); - when(decider2.canEvaluateForIndex(any())).thenReturn(false); + ConcurrentSearchRequestDecider decider1 = mock(ConcurrentSearchRequestDecider.class); - Collection concurrentSearchDeciders = new ArrayList<>(); - concurrentSearchDeciders.add(decider1); - concurrentSearchDeciders.add(decider2); + ConcurrentSearchRequestDecider decider2 = mock(ConcurrentSearchRequestDecider.class); + + ConcurrentSearchRequestDecider.Factory factory1 = new ConcurrentSearchRequestDecider.Factory() { + @Override + public Optional create(IndexSettings indexSettings) { + return Optional.ofNullable(decider1); + } + }; + + ConcurrentSearchRequestDecider.Factory factory2 = new ConcurrentSearchRequestDecider.Factory() { + @Override + public Optional create(IndexSettings indexSettings) { + return Optional.ofNullable(decider2); + } + }; + ConcurrentSearchRequestDecider.Factory factory3 = new ConcurrentSearchRequestDecider.Factory() { + @Override + public Optional create(IndexSettings indexSettings) { + return Optional.empty(); + } + }; + + List concurrentSearchRequestDeciders = new ArrayList<>(); + concurrentSearchRequestDeciders.add(factory1); + concurrentSearchRequestDeciders.add(factory2); + concurrentSearchRequestDeciders.add(factory3); context = new DefaultSearchContext( readerContext, @@ -1007,7 +1029,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - concurrentSearchDeciders + concurrentSearchRequestDeciders ); // create a supported agg operation context.aggregations(mockAggregations); @@ -1021,15 +1043,9 @@ protected Engine.Searcher acquireSearcherInternal(String source) { // Case5: multiple deciders are registered and one of them returns ConcurrentSearchDecision.DecisionStatus.NO // use non-concurrent path even if query contains supported agg - when(decider1.canEvaluateForIndex(any())).thenReturn(true); when(decider1.getConcurrentSearchDecision()).thenReturn( new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO, "disable concurrent search") ); - when(decider2.canEvaluateForIndex(any())).thenReturn(false); - - concurrentSearchDeciders.clear(); - concurrentSearchDeciders.add(decider1); - concurrentSearchDeciders.add(decider2); // create a source so that query tree is parsed by visitor SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); @@ -1051,7 +1067,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - concurrentSearchDeciders + concurrentSearchRequestDeciders ); // create a supported agg operation @@ -1067,20 +1083,17 @@ protected Engine.Searcher acquireSearcherInternal(String source) { // Case6: multiple deciders are registered and first decider returns ConcurrentSearchDecision.DecisionStatus.YES // while second decider returns ConcurrentSearchDecision.DecisionStatus.NO // use non-concurrent path even if query contains supported agg - when(decider1.canEvaluateForIndex(any())).thenReturn(true); + when(decider1.getConcurrentSearchDecision()).thenReturn( new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.YES, "enable concurrent search") ); - when(decider2.canEvaluateForIndex(any())).thenReturn(true); + when(decider2.getConcurrentSearchDecision()).thenReturn( new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO, "disable concurrent search") ); - concurrentSearchDeciders.clear(); - concurrentSearchDeciders.add(decider1); - concurrentSearchDeciders.add(decider2); - // create a source so that query tree is parsed by visitor + when(shardSearchRequest.source()).thenReturn(sourceBuilder); context = new DefaultSearchContext( readerContext, @@ -1096,7 +1109,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - concurrentSearchDeciders + concurrentSearchRequestDeciders ); // create a supported agg operation @@ -1111,22 +1124,19 @@ protected Engine.Searcher acquireSearcherInternal(String source) { // Case7: multiple deciders are registered and all return ConcurrentSearchDecision.DecisionStatus.NO_OP // but un-supported agg query is present, use non-concurrent path - when(decider1.canEvaluateForIndex(any())).thenReturn(true); + when(decider1.getConcurrentSearchDecision()).thenReturn( new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO_OP, "noop") ); - when(decider2.canEvaluateForIndex(any())).thenReturn(true); + when(decider2.getConcurrentSearchDecision()).thenReturn( new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO_OP, "noop") ); when(mockAggregations.factories().allFactoriesSupportConcurrentSearch()).thenReturn(false); - concurrentSearchDeciders.clear(); - concurrentSearchDeciders.add(decider1); - concurrentSearchDeciders.add(decider2); - // create a source so that query tree is parsed by visitor + when(shardSearchRequest.source()).thenReturn(sourceBuilder); context = new DefaultSearchContext( readerContext, @@ -1142,7 +1152,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { false, executor, null, - concurrentSearchDeciders + concurrentSearchRequestDeciders ); // create a supported agg operation diff --git a/server/src/test/java/org/opensearch/search/SearchModuleTests.java b/server/src/test/java/org/opensearch/search/SearchModuleTests.java index 71bbe7c718f47..9e8e7afe332f1 100644 --- a/server/src/test/java/org/opensearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/opensearch/search/SearchModuleTests.java @@ -41,6 +41,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; @@ -69,7 +70,7 @@ import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.aggregations.support.ValuesSourceType; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchSubPhase; import org.opensearch.search.fetch.subphase.ExplainPhase; import org.opensearch.search.fetch.subphase.highlight.CustomHighlighter; @@ -501,12 +502,12 @@ public Optional getIndexSearcherExecutorProvider() { expectThrows(IllegalStateException.class, () -> new SearchModule(Settings.EMPTY, searchPlugins)); } - public void testRegisterConcurrentSearchDecidersNoExternalPlugins() { + public void testRegisterConcurrentSearchRequestDecidersNoExternalPlugins() { SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); - assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); + assertEquals(searchModule.getConcurrentSearchRequestDeciderFactories().size(), 0); } - public void testRegisterConcurrentSearchDecidersExternalPluginsWithNoDeciders() { + public void testRegisterConcurrentSearchRequestDecidersExternalPluginsWithNoDeciders() { SearchPlugin plugin1 = new SearchPlugin() { @Override public Optional getIndexSearcherExecutorProvider() { @@ -521,10 +522,10 @@ public Optional getIndexSearcherExecutorProvider() { searchPlugins.add(plugin2); SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); - assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); + assertEquals(searchModule.getConcurrentSearchRequestDeciderFactories().size(), 0); } - public void testRegisterConcurrentSearchDecidersExternalPluginsWithDeciders() { + public void testRegisterConcurrentSearchRequestDecidersExternalPluginsWithDeciders() { SearchPlugin pluginDecider1 = new SearchPlugin() { @Override public Optional getIndexSearcherExecutorProvider() { @@ -532,15 +533,25 @@ public Optional getIndexSearcherExecutorProvider() { } @Override - public ConcurrentSearchDecider getConcurrentSearchDecider() { - return mock(ConcurrentSearchDecider.class); + public Optional getConcurrentSearchRequestDeciderFactory() { + return Optional.of(new ConcurrentSearchRequestDecider.Factory() { + @Override + public Optional create(IndexSettings indexSettings) { + return Optional.of(mock(ConcurrentSearchRequestDecider.class)); + } + }); } }; SearchPlugin pluginDecider2 = new SearchPlugin() { @Override - public ConcurrentSearchDecider getConcurrentSearchDecider() { - return mock(ConcurrentSearchDecider.class); + public Optional getConcurrentSearchRequestDeciderFactory() { + return Optional.of(new ConcurrentSearchRequestDecider.Factory() { + @Override + public Optional create(IndexSettings indexSettings) { + return Optional.of(mock(ConcurrentSearchRequestDecider.class)); + } + }); } }; @@ -549,23 +560,7 @@ public ConcurrentSearchDecider getConcurrentSearchDecider() { searchPlugins.add(pluginDecider2); SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); - assertEquals(searchModule.getConcurrentSearchDeciders().size(), 2); - } - - public void testRegisterConcurrentSearchDecidersPluginWithNullDecider() { - SearchPlugin pluginWithNullDecider = new SearchPlugin() { - @Override - public ConcurrentSearchDecider getConcurrentSearchDecider() { - return null; - } - }; - - List searchPlugins = new ArrayList<>(); - searchPlugins.add(pluginWithNullDecider); - SearchModule searchModule = new SearchModule(Settings.EMPTY, searchPlugins); - // null decider is filtered out, so 0 deciders - assertEquals(searchModule.getConcurrentSearchDeciders().size(), 0); - + assertEquals(searchModule.getConcurrentSearchRequestDeciderFactories().size(), 2); } private static final String[] NON_DEPRECATED_QUERIES = new String[] { diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index 09df9b85320f0..97c06962ca2e7 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -57,7 +57,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.search.MockSearchService; import org.opensearch.search.SearchService; -import org.opensearch.search.deciders.ConcurrentSearchDecider; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.query.QueryPhase; import org.opensearch.tasks.TaskResourceTrackingService; @@ -158,7 +158,7 @@ protected SearchService newSearchService( CircuitBreakerService circuitBreakerService, Executor indexSearcherExecutor, TaskResourceTrackingService taskResourceTrackingService, - Collection concurrentSearchDecidersList + Collection concurrentSearchDeciderFactories ) { if (getPluginsService().filterPlugins(MockSearchService.TestPlugin.class).isEmpty()) { return super.newSearchService( @@ -173,7 +173,7 @@ protected SearchService newSearchService( circuitBreakerService, indexSearcherExecutor, taskResourceTrackingService, - concurrentSearchDecidersList + concurrentSearchDeciderFactories ); } return new MockSearchService( From 95642dbf1ddbf161f561e771c8e322e51bedbefb Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 5 Sep 2024 10:36:17 -0700 Subject: [PATCH 064/357] Update newQueryShardContext() overload to fix breaking changes (#15710) Signed-off-by: Harsha Vamsi Kalluri --- .../org/opensearch/index/IndexService.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index e5dd44a70a11c..f1b36194bf62d 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -867,7 +867,7 @@ public IndexSettings getIndexSettings() { * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { - return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, false, false); + return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, false); } /** @@ -913,6 +913,22 @@ public QueryShardContext newQueryShardContext( ); } + /** + * Creates a new QueryShardContext. + *

+ * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make + * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. + */ + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + boolean validate + ) { + return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, validate, false); + } + /** * The {@link ThreadPool} to use for this index. */ From 6f2ea3928f8af116fdbf023828681bbd02911ef6 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Thu, 5 Sep 2024 23:21:03 +0530 Subject: [PATCH 065/357] Making _cat/allocation API use indexLevelStats (#15292) Signed-off-by: Harsh Garg Co-authored-by: Harsh Garg --- CHANGELOG.md | 1 + .../org/opensearch/rest/action/cat/RestAllocationAction.java | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c0c78d6db02..5f48c75af6226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) +- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java index 07b0fbbe4a911..912e5173f8e01 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java @@ -101,6 +101,7 @@ public void processResponse(final ClusterStateResponse state) { statsRequest.clear() .addMetric(NodesStatsRequest.Metric.FS.metricName()) .indices(new CommonStatsFlags(CommonStatsFlags.Flag.Store)); + statsRequest.indices().setIncludeIndicesStatsByLevel(true); client.admin().cluster().nodesStats(statsRequest, new RestResponseListener(channel) { @Override From 2eb148cdffd32058c40d6703cbb4a06eb2a2cba3 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 5 Sep 2024 23:30:58 +0530 Subject: [PATCH 066/357] Integ Test for RemotePublication can be disabled by rolling restart of masters (#15677) Signed-off-by: Shivansh Arora --- .../remote/RemoteStatePublicationIT.java | 133 +++++++++++++++++- .../opensearch/test/InternalTestCluster.java | 11 ++ 2 files changed, 137 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index f794eec65cb1b..0778782c5bec5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -8,14 +8,17 @@ package org.opensearch.gateway.remote; -import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationState; +import org.opensearch.cluster.coordination.PersistedStateRegistry; +import org.opensearch.cluster.coordination.PublishClusterStateStats; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.discovery.DiscoveryStats; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; @@ -27,6 +30,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; import org.junit.Before; @@ -34,11 +38,19 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.HashSet; import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; +import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.DISCOVERY; +import static org.opensearch.cluster.metadata.Metadata.isGlobalStateEquals; +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -77,10 +89,7 @@ public void setup() { @Override protected Settings featureFlagSettings() { - return Settings.builder() - .put(super.featureFlagSettings()) - .put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled) - .build(); + return Settings.builder().put(super.featureFlagSettings()).put(REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled).build(); } @Override @@ -220,11 +229,121 @@ public void testRemotePublicationDownloadStats() { NodesStatsResponse nodesStatsResponseDataNode = client().admin() .cluster() .prepareNodesStats(dataNode) - .addMetric(NodesStatsRequest.Metric.DISCOVERY.metricName()) + .addMetric(DISCOVERY.metricName()) .get(); assertDataNodeDownloadStats(nodesStatsResponseDataNode); + } + + public void testRemotePublicationDisabledByRollingRestart() throws Exception { + prepareCluster(3, 2, INDEX_NAME, 1, 2); + ensureStableCluster(5); + ensureGreen(INDEX_NAME); + + Set clusterManagers = internalCluster().getClusterManagerNames(); + Set restartedMasters = new HashSet<>(); + + for (String clusterManager : clusterManagers) { + internalCluster().restartNode(clusterManager, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + restartedMasters.add(nodeName); + return Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, false).build(); + } + + @Override + public void doAfterNodes(int n, Client client) { + String activeCM = internalCluster().getClusterManagerName(); + Set followingCMs = clusterManagers.stream() + .filter(node -> !Objects.equals(node, activeCM)) + .collect(Collectors.toSet()); + boolean activeCMRestarted = restartedMasters.contains(activeCM); + NodesStatsResponse response = client().admin() + .cluster() + .prepareNodesStats(followingCMs.toArray(new String[0])) + .clear() + .addMetric(DISCOVERY.metricName()) + .get(); + // after master is flipped to restarted master, publication should happen on Transport + response.getNodes().forEach(nodeStats -> { + if (activeCMRestarted) { + PublishClusterStateStats stats = nodeStats.getDiscoveryStats().getPublishStats(); + assertTrue( + stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0 + ); + assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); + } else { + DiscoveryStats stats = nodeStats.getDiscoveryStats(); + assertEquals(0, stats.getPublishStats().getFullClusterStateReceivedCount()); + assertEquals(0, stats.getPublishStats().getCompatibleClusterStateDiffReceivedCount()); + assertEquals(0, stats.getPublishStats().getIncompatibleClusterStateDiffReceivedCount()); + } + }); + + NodesInfoResponse nodesInfoResponse = client().admin() + .cluster() + .prepareNodesInfo(activeCM) + .clear() + .addMetric(SETTINGS.metricName()) + .get(); + // if masterRestarted is true Publication Setting should be false, and vice versa + assertTrue( + REMOTE_PUBLICATION_EXPERIMENTAL_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings()) != activeCMRestarted + ); + + followingCMs.forEach(node -> { + PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); + CoordinationState.PersistedState remoteState = registry.getPersistedState( + PersistedStateRegistry.PersistedStateType.REMOTE + ); + if (activeCMRestarted) { + assertNull(remoteState.getLastAcceptedState()); + // assertNull(remoteState.getLastAcceptedManifest()); + } else { + ClusterState localState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL) + .getLastAcceptedState(); + ClusterState remotePersistedState = remoteState.getLastAcceptedState(); + assertTrue(isGlobalStateEquals(localState.metadata(), remotePersistedState.metadata())); + assertEquals(localState.nodes(), remotePersistedState.nodes()); + assertEquals(localState.routingTable(), remotePersistedState.routingTable()); + assertEquals(localState.customs(), remotePersistedState.customs()); + } + }); + } + }); + + } + ensureGreen(INDEX_NAME); + ensureStableCluster(5); + + String activeCM = internalCluster().getClusterManagerName(); + Set followingCMs = clusterManagers.stream().filter(node -> !Objects.equals(node, activeCM)).collect(Collectors.toSet()); + NodesStatsResponse response = client().admin() + .cluster() + .prepareNodesStats(followingCMs.toArray(new String[0])) + .clear() + .addMetric(DISCOVERY.metricName()) + .get(); + response.getNodes().forEach(nodeStats -> { + PublishClusterStateStats stats = nodeStats.getDiscoveryStats().getPublishStats(); + assertTrue(stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0); + assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); + }); + NodesInfoResponse nodesInfoResponse = client().admin() + .cluster() + .prepareNodesInfo(activeCM) + .clear() + .addMetric(SETTINGS.metricName()) + .get(); + // if masterRestarted is true Publication Setting should be false, and vice versa + assertFalse(REMOTE_PUBLICATION_EXPERIMENTAL_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings())); + followingCMs.forEach(node -> { + PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); + CoordinationState.PersistedState remoteState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); + assertNull(remoteState.getLastAcceptedState()); + // assertNull(remoteState.getLastAcceptedManifest()); + }); } private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) { diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index ec88002317284..7adff82e72245 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2143,6 +2143,17 @@ public synchronized void fullRestart(RestartCallback callback) throws Exception } } + /** + * Returns the name of all the cluster managers in the cluster + */ + public Set getClusterManagerNames() { + return nodes.entrySet() + .stream() + .filter(entry -> CLUSTER_MANAGER_NODE_PREDICATE.test(entry.getValue())) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + } + /** * Returns the name of the current cluster-manager node in the cluster. */ From e5e8504acdf1339f53dced4c00f5c9cd91b95f2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:04:01 -0400 Subject: [PATCH 067/357] Bump com.azure:azure-identity from 1.13.0 to 1.13.2 in /plugins/repository-azure (#15578) * Bump com.azure:azure-identity in /plugins/repository-azure Bumps [com.azure:azure-identity](https://github.com/Azure/azure-sdk-for-java) from 1.13.0 to 1.13.2. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core_1.13.0...azure-identity_1.13.2) --- updated-dependencies: - dependency-name: com.azure:azure-identity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../repository-azure/licenses/azure-identity-1.13.0.jar.sha1 | 1 - .../repository-azure/licenses/azure-identity-1.13.2.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-identity-1.13.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f48c75af6226..1b4b729152b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419)) - Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) - Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) +- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) ### Changed - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index e76556f24cc23..c8767478e8dad 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -57,7 +57,7 @@ dependencies { api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') api 'com.azure:azure-storage-blob:12.23.0' - api 'com.azure:azure-identity:1.13.0' + api 'com.azure:azure-identity:1.13.2' // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" diff --git a/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1 deleted file mode 100644 index b59c2a3be5c92..0000000000000 --- a/plugins/repository-azure/licenses/azure-identity-1.13.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -54b44a74636322d06e9dc42d611a9f12a0966790 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-identity-1.13.2.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.13.2.jar.sha1 new file mode 100644 index 0000000000000..7c98a9ccba592 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-identity-1.13.2.jar.sha1 @@ -0,0 +1 @@ +50a1daef3eb5c6ab2e1351a3e3f5a7649a8fe464 \ No newline at end of file From 245fa4703d1b61bfa07f881e82d2ff9ac8a2e064 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 6 Sep 2024 00:19:06 +0530 Subject: [PATCH 068/357] Add path prefix support to hashed prefix snapshots (#15664) * Add path prefix support to hashed prefix snapshots Signed-off-by: Ashish Singh * Fix test Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../common/settings/ClusterSettings.java | 2 ++ .../blobstore/BlobStoreRepository.java | 14 ++++++++++++++ .../opensearch/test/OpenSearchIntegTestCase.java | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4b729152b12..a7f3161daff23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) +- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664)) - Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index c6a3ee6df609b..e35aa42563d4d 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -152,6 +152,7 @@ import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlSettings; import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings; import org.opensearch.ratelimitting.admissioncontrol.settings.IoBasedAdmissionControllerSettings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.rest.BaseRestHandler; import org.opensearch.script.ScriptService; @@ -777,6 +778,7 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, + BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 01d924aa17839..b954560c1bc94 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -343,6 +343,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.NodeScope ); + /** + * Controls the fixed prefix for the snapshot shard blob path. + */ + public static final Setting SNAPSHOT_SHARD_PATH_PREFIX_SETTING = Setting.simpleString( + "cluster.snapshot.shard.path.prefix", + "", + Setting.Property.NodeScope, + Setting.Property.Final + ); + protected volatile boolean supportURLRepo; private volatile int maxShardBlobDeleteBatch; @@ -434,6 +444,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final NamedXContentRegistry namedXContentRegistry; + private final String snapshotShardPathPrefix; + /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart @@ -485,6 +497,7 @@ protected BlobStoreRepository( this.clusterService = clusterService; this.recoverySettings = recoverySettings; this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()); + this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings()); } @Override @@ -2729,6 +2742,7 @@ private BlobPath shardPath(IndexId indexId, int shardId) { SnapshotShardPathInput shardPathInput = new SnapshotShardPathInput.Builder().basePath(basePath()) .indexUUID(indexId.getId()) .shardId(String.valueOf(shardId)) + .fixedPrefix(snapshotShardPathPrefix) .build(); PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null; return pathType.path(shardPathInput, pathHashAlgorithm); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 7d1fe1b000d20..e474ef202b235 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -403,6 +403,8 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static Boolean segmentsPathFixedPrefix; + private static Boolean snapshotShardPathFixedPrefix; + private Path remoteStoreRepositoryPath; private ReplicationType randomReplicationType; @@ -414,6 +416,7 @@ public static void beforeClass() throws Exception { prefixModeVerificationEnable = randomBoolean(); translogPathFixedPrefix = randomBoolean(); segmentsPathFixedPrefix = randomBoolean(); + snapshotShardPathFixedPrefix = randomBoolean(); testClusterRule.beforeClass(); } @@ -2901,6 +2904,7 @@ private static Settings buildRemoteStoreNodeAttributes( settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : ""); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : ""); + settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), segmentsPathFixedPrefix ? "c" : ""); return settings.build(); } From f5c897cf1db3d413c6ccd9bcad1284a7873a1556 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:59:24 -0700 Subject: [PATCH 069/357] Adding WithFieldName interface for QueryBuilders with fieldName (#15705) Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../index/query/CorrelationQueryBuilder.java | 4 +++- .../query/AbstractGeometryQueryBuilder.java | 5 ++++- .../index/query/BaseTermQueryBuilder.java | 3 ++- .../index/query/CommonTermsQueryBuilder.java | 3 ++- .../query/DistanceFeatureQueryBuilder.java | 5 +++-- .../index/query/ExistsQueryBuilder.java | 3 ++- .../query/FieldMaskingSpanQueryBuilder.java | 6 +++++- .../query/GeoBoundingBoxQueryBuilder.java | 3 ++- .../index/query/GeoDistanceQueryBuilder.java | 3 ++- .../index/query/GeoPolygonQueryBuilder.java | 3 ++- .../query/MatchBoolPrefixQueryBuilder.java | 3 ++- .../query/MatchPhrasePrefixQueryBuilder.java | 3 ++- .../index/query/MatchPhraseQueryBuilder.java | 3 ++- .../index/query/MatchQueryBuilder.java | 3 ++- .../index/query/MultiTermQueryBuilder.java | 7 +------ .../index/query/SpanNearQueryBuilder.java | 3 ++- .../index/query/TermsQueryBuilder.java | 3 ++- .../opensearch/index/query/WithFieldName.java | 21 +++++++++++++++++++ 19 files changed, 62 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/query/WithFieldName.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a7f3161daff23..06a749df1b3a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) +- Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java index 806ac0389b5f3..e95b68e855cca 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.WithFieldName; import org.opensearch.plugin.correlation.core.index.mapper.VectorFieldMapper; import java.io.IOException; @@ -36,7 +37,7 @@ * * @opensearch.internal */ -public class CorrelationQueryBuilder extends AbstractQueryBuilder { +public class CorrelationQueryBuilder extends AbstractQueryBuilder implements WithFieldName { private static final Logger log = LogManager.getLogger(CorrelationQueryBuilder.class); protected static final ParseField VECTOR_FIELD = new ParseField("vector"); @@ -205,6 +206,7 @@ public void setFieldName(String fieldName) { * get field name * @return field name */ + @Override public String fieldName() { return fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java index 9fb857e33bfee..3823b524df6fe 100644 --- a/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/AbstractGeometryQueryBuilder.java @@ -67,7 +67,9 @@ * * @opensearch.internal */ -public abstract class AbstractGeometryQueryBuilder> extends AbstractQueryBuilder { +public abstract class AbstractGeometryQueryBuilder> extends AbstractQueryBuilder + implements + WithFieldName { public static final String DEFAULT_SHAPE_INDEX_NAME = "shapes"; public static final String DEFAULT_SHAPE_FIELD_NAME = "shape"; @@ -218,6 +220,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { /** * @return the name of the field that will be queried */ + @Override public String fieldName() { return fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java index c4d9437a60c75..deb36a26e4c86 100644 --- a/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BaseTermQueryBuilder.java @@ -47,7 +47,7 @@ * * @opensearch.internal */ -public abstract class BaseTermQueryBuilder> extends AbstractQueryBuilder { +public abstract class BaseTermQueryBuilder> extends AbstractQueryBuilder implements WithFieldName { public static final ParseField VALUE_FIELD = new ParseField("value"); @@ -153,6 +153,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Returns the field name used in this query. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java index 652cae86da0dc..24b10851cbe10 100644 --- a/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/CommonTermsQueryBuilder.java @@ -67,7 +67,7 @@ * @opensearch.internal */ @Deprecated -public class CommonTermsQueryBuilder extends AbstractQueryBuilder { +public class CommonTermsQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String COMMON_TERMS_QUERY_DEPRECATION_MSG = "[match] query which can efficiently " + "skip blocks of documents if the total number of hits is not tracked"; @@ -152,6 +152,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeFloat(cutoffFrequency); } + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java index 1d9f0479c6b17..e4f3d8556158a 100644 --- a/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/DistanceFeatureQueryBuilder.java @@ -57,7 +57,7 @@ * * @opensearch.internal */ -public class DistanceFeatureQueryBuilder extends AbstractQueryBuilder { +public class DistanceFeatureQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "distance_feature"; private static final ParseField FIELD_FIELD = new ParseField("field"); @@ -136,7 +136,8 @@ protected Query doToQuery(QueryShardContext context) throws IOException { return fieldType.distanceFeatureQuery(origin.origin(), pivot, 1.0f, context); } - String fieldName() { + @Override + public String fieldName() { return field; } diff --git a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java index 6ae40fe1b1e64..f6b24e98f0f71 100644 --- a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java @@ -59,7 +59,7 @@ * * @opensearch.internal */ -public class ExistsQueryBuilder extends AbstractQueryBuilder { +public class ExistsQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "exists"; public static final ParseField FIELD_FIELD = new ParseField("field"); @@ -89,6 +89,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { /** * @return the field name that has to exist for this query to match */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java index 4e73d87b07b7a..7846098b55071 100644 --- a/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -53,7 +53,10 @@ * * @opensearch.internal */ -public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder implements SpanQueryBuilder { +public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder + implements + SpanQueryBuilder, + WithFieldName { public static final String NAME = "span_field_masking"; public static final ParseField SPAN_FIELD_MASKING_FIELD = new ParseField(NAME, "field_masking_span"); @@ -100,6 +103,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { /** * @return the field name for this query */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java index 1fade8601e2a6..52e96159cf06e 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -66,7 +66,7 @@ * * @opensearch.internal * */ -public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder { +public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "geo_bounding_box"; /** Default type for executing this query (memory as of this writing). */ @@ -263,6 +263,7 @@ public GeoExecType type() { } /** Returns the name of the field to base the bounding box computation on. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java index 8d126f19a204c..79377ba01701f 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoDistanceQueryBuilder.java @@ -63,7 +63,7 @@ * * @opensearch.internal */ -public class GeoDistanceQueryBuilder extends AbstractQueryBuilder { +public class GeoDistanceQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "geo_distance"; /** Default for distance unit computation. */ @@ -129,6 +129,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Name of the field this query is operating on. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java index 47eafa3893384..e120f04ed9351 100644 --- a/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/GeoPolygonQueryBuilder.java @@ -61,7 +61,7 @@ * * @opensearch.internal */ -public class GeoPolygonQueryBuilder extends AbstractQueryBuilder { +public class GeoPolygonQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "geo_polygon"; /** @@ -131,6 +131,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBoolean(ignoreUnmapped); } + @Override public String fieldName() { return fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java index 7ceb17203e837..2c2c2de943e2f 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilder.java @@ -61,7 +61,7 @@ * * @opensearch.internal */ -public class MatchBoolPrefixQueryBuilder extends AbstractQueryBuilder { +public class MatchBoolPrefixQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "match_bool_prefix"; @@ -127,6 +127,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Returns the field name used in this query. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java index d61a5957627ea..3337a31658ca1 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchPhrasePrefixQueryBuilder.java @@ -52,7 +52,7 @@ * * @opensearch.internal */ -public class MatchPhrasePrefixQueryBuilder extends AbstractQueryBuilder { +public class MatchPhrasePrefixQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "match_phrase_prefix"; public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); @@ -104,6 +104,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Returns the field name used in this query. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java index 6cdf6c6600304..97e03d53d38f1 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchPhraseQueryBuilder.java @@ -52,7 +52,7 @@ * * @opensearch.internal */ -public class MatchPhraseQueryBuilder extends AbstractQueryBuilder { +public class MatchPhraseQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "match_phrase"; public static final ParseField SLOP_FIELD = new ParseField("slop"); public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); @@ -100,6 +100,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Returns the field name used in this query. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java index 5e9e6a3660e76..593077d18951e 100644 --- a/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MatchQueryBuilder.java @@ -56,7 +56,7 @@ * * @opensearch.internal */ -public class MatchQueryBuilder extends AbstractQueryBuilder { +public class MatchQueryBuilder extends AbstractQueryBuilder implements WithFieldName { private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "you can omit this option, " + "the [match] query can skip block of documents efficiently if the total number of hits is not tracked"; @@ -171,6 +171,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } /** Returns the field name used in this query. */ + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/MultiTermQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MultiTermQueryBuilder.java index b8e88da4741bb..f854df79f1ee8 100644 --- a/server/src/main/java/org/opensearch/index/query/MultiTermQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MultiTermQueryBuilder.java @@ -36,9 +36,4 @@ * * @opensearch.internal */ -public interface MultiTermQueryBuilder extends QueryBuilder { - /** - * Get the field name for this query. - */ - String fieldName(); -} +public interface MultiTermQueryBuilder extends QueryBuilder, WithFieldName {} diff --git a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java index 30a1c29c29126..179673f500a92 100644 --- a/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/SpanNearQueryBuilder.java @@ -322,7 +322,7 @@ public void visit(QueryBuilderVisitor visitor) { * * @opensearch.internal */ - public static class SpanGapQueryBuilder implements SpanQueryBuilder { + public static class SpanGapQueryBuilder implements SpanQueryBuilder, WithFieldName { public static final String NAME = "span_gap"; /** Name of field to match against. */ @@ -358,6 +358,7 @@ public SpanGapQueryBuilder(StreamInput in) throws IOException { /** * @return fieldName The name of the field */ + @Override public String fieldName() { return fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java index 4b92d6a1f5460..dbd141e00f81c 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java @@ -79,7 +79,7 @@ * * @opensearch.internal */ -public class TermsQueryBuilder extends AbstractQueryBuilder { +public class TermsQueryBuilder extends AbstractQueryBuilder implements WithFieldName { public static final String NAME = "terms"; private final String fieldName; @@ -269,6 +269,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } } + @Override public String fieldName() { return this.fieldName; } diff --git a/server/src/main/java/org/opensearch/index/query/WithFieldName.java b/server/src/main/java/org/opensearch/index/query/WithFieldName.java new file mode 100644 index 0000000000000..adfd789ac3707 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/query/WithFieldName.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +/** + * Interface for classes with a fieldName method + * + * @opensearch.internal + */ +public interface WithFieldName { + /** + * Get the field name for this query. + */ + String fieldName(); +} From 8f34ce5ca3849687d509205bfe4378e1183eb6fb Mon Sep 17 00:00:00 2001 From: Finn Date: Thu, 5 Sep 2024 12:01:43 -0700 Subject: [PATCH 070/357] Mitigation for remote snapshot filecache overflow (#15077) TransferManager fails BlobFetchRequest on full cache Signed-off-by: Finn Carroll --- CHANGELOG.md | 1 + .../store/remote/utils/TransferManager.java | 13 +++++++ .../remote/utils/TransferManagerTestCase.java | 39 ++++++++----------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a749df1b3a7..409e152ac60af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) - Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) - Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) +- Fix remote snapshot file_cache exceeding capacity ([#15077](https://github.com/opensearch-project/OpenSearch/pull/15077)) ### Security diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java index f07c4832d982c..94c25202ac90c 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java @@ -95,6 +95,19 @@ public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOExceptio @SuppressWarnings("removal") private static FileCachedIndexInput createIndexInput(FileCache fileCache, StreamReader streamReader, BlobFetchRequest request) { try { + // This local file cache is ref counted and may not strictly enforce configured capacity. + // If we find available capacity is exceeded, deny further BlobFetchRequests. + if (fileCache.capacity() < fileCache.usage().usage()) { + fileCache.prune(); + throw new IOException( + "Local file cache capacity (" + + fileCache.capacity() + + ") exceeded (" + + fileCache.usage().usage() + + ") - BlobFetchRequest failed: " + + request.getFilePath() + ); + } if (Files.exists(request.getFilePath()) == false) { logger.trace("Fetching from Remote in createIndexInput of Transfer Manager"); try ( diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java index 810a4c336fdf7..1eae5119ab462 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java @@ -99,7 +99,7 @@ public void testConcurrentAccess() throws Exception { } } - public void testFetchBlobWithConcurrentCacheEvictions() throws Exception { + public void testFetchBlobWithConcurrentCacheEvictions() { // Submit 256 tasks to an executor with 16 threads that will each randomly // request one of eight blobs. Given that the cache can only hold two // blobs this will lead to a huge amount of contention and thrashing. @@ -114,41 +114,34 @@ public void testFetchBlobWithConcurrentCacheEvictions() throws Exception { try (IndexInput indexInput = fetchBlobWithName(blobname)) { assertIndexInputIsFunctional(indexInput); } + } catch (IOException ignored) { // fetchBlobWithName may fail due to fixed capacity } catch (Exception e) { throw new AssertionError(e); } })); } // Wait for all threads to complete - for (Future future : futures) { - future.get(10, TimeUnit.SECONDS); + try { + for (Future future : futures) { + future.get(10, TimeUnit.SECONDS); + } + } catch (java.util.concurrent.ExecutionException ignored) { // Index input may be null + } catch (Exception e) { + throw new AssertionError(e); } + } finally { assertTrue(terminate(testRunner)); } MatcherAssert.assertThat("Expected many evictions to happen", fileCache.stats().evictionCount(), greaterThan(0L)); } - public void testUsageExceedsCapacity() throws Exception { - // Fetch resources that exceed the configured capacity of the cache and assert that the - // returned IndexInputs are still functional. - try (IndexInput i1 = fetchBlobWithName("1"); IndexInput i2 = fetchBlobWithName("2"); IndexInput i3 = fetchBlobWithName("3")) { - assertIndexInputIsFunctional(i1); - assertIndexInputIsFunctional(i2); - assertIndexInputIsFunctional(i3); - MatcherAssert.assertThat(fileCache.usage().activeUsage(), equalTo((long) EIGHT_MB * 3)); - MatcherAssert.assertThat(fileCache.usage().usage(), equalTo((long) EIGHT_MB * 3)); - } - MatcherAssert.assertThat(fileCache.usage().activeUsage(), equalTo(0L)); - MatcherAssert.assertThat(fileCache.usage().usage(), equalTo((long) EIGHT_MB * 3)); - // Fetch another resource which will trigger an eviction - try (IndexInput i1 = fetchBlobWithName("1")) { - assertIndexInputIsFunctional(i1); - MatcherAssert.assertThat(fileCache.usage().activeUsage(), equalTo((long) EIGHT_MB)); - MatcherAssert.assertThat(fileCache.usage().usage(), equalTo((long) EIGHT_MB)); - } - MatcherAssert.assertThat(fileCache.usage().activeUsage(), equalTo(0L)); - MatcherAssert.assertThat(fileCache.usage().usage(), equalTo((long) EIGHT_MB)); + public void testOverflowDisabled() throws Exception { + initializeTransferManager(); + IndexInput i1 = fetchBlobWithName("1"); + IndexInput i2 = fetchBlobWithName("2"); + + assertThrows(IOException.class, () -> { IndexInput i3 = fetchBlobWithName("3"); }); } public void testDownloadFails() throws Exception { From 375dda3e07b2c37450c4cb19b6722e04a1ce64f5 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 6 Sep 2024 00:41:54 +0530 Subject: [PATCH 071/357] Mute flaky test RemoteFsTimestampAwareTranslogTests.testSimpleOperationsUpload (#15732) Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../index/translog/RemoteFsTimestampAwareTranslogTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index c510a6475147d..4c9da7e95dfa7 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -310,6 +310,7 @@ public void testIndexDeletionWithNoPinnedTimestampButRecentFiles() throws Except } @Override + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15731") public void testSimpleOperationsUpload() throws Exception { ArrayList ops = new ArrayList<>(); From fe61e4f299d4b832dc232cb223ec946585177f52 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 5 Sep 2024 16:42:45 -0400 Subject: [PATCH 072/357] Fixing MacOS 13 assemble workflows (#15747) Signed-off-by: Andriy Redko --- .github/workflows/assemble.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index 294627622a136..b3838b8e5ae97 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -30,8 +30,11 @@ jobs: - name: Setup docker (missing on MacOS) id: setup_docker if: runner.os == 'macos' + continue-on-error: true run: | - exit 0; + brew install docker colima coreutils + gtimeout 15m colima start + shell: bash - name: Run Gradle (assemble) if: runner.os == 'macos' && steps.setup_docker.outcome != 'success' run: | @@ -45,4 +48,4 @@ jobs: - name: Run Gradle (assemble) if: runner.os == 'macos' && steps.setup_docker.outcome == 'success' run: | - exit 0; + ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} From 3fdbd8b7308149d5432f8d04b8040e15a934c906 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 6 Sep 2024 07:25:02 +0530 Subject: [PATCH 073/357] Make Remote Publication a static setting (#15478) * Make Remote Publication a static setting Signed-off-by: Shivansh Arora --- CHANGELOG.md | 1 + .../RemoteClusterStateCleanupManagerIT.java | 4 ++-- .../remote/RemoteRoutingTableServiceIT.java | 4 ++-- .../remote/RemoteStatePublicationIT.java | 18 ++++++------------ .../coordination/CoordinationState.java | 7 +++---- .../common/settings/ClusterSettings.java | 1 + .../common/settings/FeatureFlagSettings.java | 1 - .../opensearch/common/util/FeatureFlags.java | 12 ------------ .../remote/RemoteClusterStateService.java | 16 +++++++++++++--- .../coordination/CoordinationStateTests.java | 4 ++-- .../RemoteRoutingTableServiceFactoryTests.java | 6 ++---- .../remote/RemoteRoutingTableServiceTests.java | 6 ++---- .../remote/RemoteClusterStateServiceTests.java | 18 ++++++------------ 13 files changed, 40 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 409e152ac60af..26358d269a2ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) +- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index 47ec3f25bcd64..cf17a58d937de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -35,12 +35,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteUploadStats.REMOTE_UPLOAD; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; @@ -189,7 +189,7 @@ public void testRemoteCleanupDeleteStaleIndexRoutingFiles() throws Exception { RemoteStoreEnums.PathType.HASHED_PREFIX.toString() ) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) - .put(REMOTE_PUBLICATION_EXPERIMENTAL, true); + .put(REMOTE_PUBLICATION_SETTING_KEY, true); int shardCount = randomIntBetween(1, 2); int replicaCount = 1; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java index 0a8c13adb034f..d143cbd7c3450 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java @@ -37,8 +37,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_TABLE; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -66,7 +66,7 @@ protected Settings nodeSettings(int nodeOrdinal) { RemoteStoreEnums.PathType.HASHED_PREFIX.toString() ) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) - .put(REMOTE_PUBLICATION_EXPERIMENTAL, true) + .put(REMOTE_PUBLICATION_SETTING_KEY, true) .put( RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 0778782c5bec5..7e7d3060ec7a9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -49,10 +49,10 @@ import static org.opensearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.DISCOVERY; import static org.opensearch.cluster.metadata.Metadata.isGlobalStateEquals; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; @@ -87,11 +87,6 @@ public void setup() { hasRemoteRoutingCharPrefix = randomBoolean(); } - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled).build(); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { String routingTableRepoName = "remote-routing-repo"; @@ -109,6 +104,7 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled) + .put(REMOTE_PUBLICATION_SETTING_KEY, isRemotePublicationEnabled) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) @@ -248,7 +244,7 @@ public void testRemotePublicationDisabledByRollingRestart() throws Exception { @Override public Settings onNodeStopped(String nodeName) { restartedMasters.add(nodeName); - return Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, false).build(); + return Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, false).build(); } @Override @@ -287,9 +283,7 @@ public void doAfterNodes(int n, Client client) { .addMetric(SETTINGS.metricName()) .get(); // if masterRestarted is true Publication Setting should be false, and vice versa - assertTrue( - REMOTE_PUBLICATION_EXPERIMENTAL_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings()) != activeCMRestarted - ); + assertTrue(REMOTE_PUBLICATION_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings()) != activeCMRestarted); followingCMs.forEach(node -> { PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); @@ -336,7 +330,7 @@ public void doAfterNodes(int n, Client client) { .addMetric(SETTINGS.metricName()) .get(); // if masterRestarted is true Publication Setting should be false, and vice versa - assertFalse(REMOTE_PUBLICATION_EXPERIMENTAL_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings())); + assertFalse(REMOTE_PUBLICATION_SETTING.get(nodesInfoResponse.getNodes().get(0).getSettings())); followingCMs.forEach(node -> { PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index c7820c2c9a365..9c883175e3ee0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -39,7 +39,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import java.io.Closeable; @@ -53,7 +52,7 @@ import java.util.Set; import static org.opensearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -81,7 +80,7 @@ public class CoordinationState { private VotingConfiguration lastPublishedConfiguration; private VoteCollection publishVotes; private final boolean isRemoteStateEnabled; - private final boolean isRemotePublicationEnabled; + private boolean isRemotePublicationEnabled; public CoordinationState( DiscoveryNode localNode, @@ -106,7 +105,7 @@ public CoordinationState( this.publishVotes = new VoteCollection(); this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); this.isRemotePublicationEnabled = isRemoteStateEnabled - && FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) + && REMOTE_PUBLICATION_SETTING.get(settings) && localNode.isRemoteStatePublicationEnabled(); } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index e35aa42563d4d..09832e2b41b6d 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -734,6 +734,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, + RemoteClusterStateService.REMOTE_PUBLICATION_SETTING, INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 9c7684923d06c..c8d00f65bda10 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,7 +37,6 @@ protected FeatureFlagSettings( FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.PLUGGABLE_CACHE_SETTING, - FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 0fd5edde2b94c..49ecbb0a7069d 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -67,11 +67,6 @@ public class FeatureFlags { */ public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; - /** - * Gates the functionality of remote routing table. - */ - public static final String REMOTE_PUBLICATION_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.publication.enabled"; - /** * Gates the functionality of background task execution. */ @@ -101,12 +96,6 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); - public static final Setting REMOTE_PUBLICATION_EXPERIMENTAL_SETTING = Setting.boolSetting( - REMOTE_PUBLICATION_EXPERIMENTAL, - false, - Property.NodeScope - ); - public static final Setting READER_WRITER_SPLIT_EXPERIMENTAL_SETTING = Setting.boolSetting( READER_WRITER_SPLIT_EXPERIMENTAL, false, @@ -148,7 +137,6 @@ public class FeatureFlags { DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, PLUGGABLE_CACHE_SETTING, - REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, READER_WRITER_SPLIT_EXPERIMENTAL_SETTING diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index fe34b68702c41..3425550a9f548 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -40,7 +40,6 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -92,7 +91,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static org.opensearch.cluster.ClusterState.CUSTOM_VALUE_SERIALIZER; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V3; @@ -123,6 +121,18 @@ public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); + /** + * Gates the functionality of remote publication. + */ + public static final String REMOTE_PUBLICATION_SETTING_KEY = "cluster.remote_store.publication.enabled"; + + public static final Setting REMOTE_PUBLICATION_SETTING = Setting.boolSetting( + REMOTE_PUBLICATION_SETTING_KEY, + false, + Property.NodeScope, + Property.Final + ); + /** * Used to specify if cluster state metadata should be published to remote store */ @@ -260,7 +270,7 @@ public RemoteClusterStateService( this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; this.indexMetadataUploadListeners = indexMetadataUploadListeners; - this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) + this.isPublicationEnabled = REMOTE_PUBLICATION_SETTING.get(settings) && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings) && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings); this.remotePathPrefix = CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.get(settings); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index ee9a2951ec541..d003b54adcccc 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -66,9 +66,9 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -1010,7 +1010,7 @@ public void testIsRemotePublicationEnabled_WithInconsistentSettings() { // create settings with remote state disabled but publication enabled Settings settings = Settings.builder() .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), false) - .put(REMOTE_PUBLICATION_EXPERIMENTAL, true) + .put(REMOTE_PUBLICATION_SETTING_KEY, true) .build(); CoordinationState coordinationState = createCoordinationState(psr1, node1, settings); assertFalse(coordinationState.isRemotePublicationEnabled()); diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java index 86f4b9502d6ab..683942fd34a37 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactoryTests.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; @@ -20,7 +19,7 @@ import java.util.function.Supplier; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteRoutingTableServiceFactoryTests extends OpenSearchTestCase { @@ -50,9 +49,8 @@ public void testGetServiceWhenRemoteRoutingEnabled() { Settings settings = Settings.builder() .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .put(FsRepository.REPOSITORIES_COMPRESS_SETTING.getKey(), false) + .put(REMOTE_PUBLICATION_SETTING_KEY, "true") .build(); - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); RemoteRoutingTableService service = RemoteRoutingTableServiceFactory.getService( repositoriesService, settings, diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 5061de9161ab4..63501f878d55d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -28,7 +28,6 @@ import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.TestCapturingListener; import org.opensearch.core.action.ActionListener; import org.opensearch.core.compress.Compressor; @@ -63,8 +62,8 @@ import org.mockito.Mockito; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.ClusterMetadataManifestTests.randomUploadedIndexMetadataList; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -114,6 +113,7 @@ public void setup() { Settings settings = Settings.builder() .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put(REMOTE_PUBLICATION_SETTING_KEY, "true") .build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); clusterService = mock(ClusterService.class); @@ -126,8 +126,6 @@ public void setup() { when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); when(blobStoreRepository.blobStore()).thenReturn(blobStore); when(blobStore.blobContainer(any())).thenReturn(blobContainer); - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); compressor = new NoneCompressor(); basePath = BlobPath.cleanPath().add("base-path"); when(blobStoreRepository.basePath()).thenReturn(basePath); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 21b88e5bd66b9..608cc2e12b055 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -107,12 +107,12 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; -import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata1; import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata2; import static org.opensearch.gateway.remote.RemoteClusterStateTestUtils.CustomMetadata3; @@ -273,8 +273,6 @@ public void teardown() throws Exception { super.tearDown(); remoteClusterStateService.close(); publicationEnabled = false; - Settings nodeSettings = Settings.builder().build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); threadPool.shutdown(); } @@ -371,8 +369,7 @@ public void testWriteFullMetadataSuccess() throws IOException { public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException { // TODO Make the publication flag parameterized publicationEnabled = true; - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, publicationEnabled).build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); + settings = Settings.builder().put(settings).put(REMOTE_PUBLICATION_SETTING_KEY, publicationEnabled).build(); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, @@ -749,8 +746,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws IOException { publicationEnabled = true; - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, publicationEnabled).build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); + settings = Settings.builder().put(settings).put(REMOTE_PUBLICATION_SETTING_KEY, true).build(); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, @@ -2658,7 +2654,7 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { .build(); clusterSettings.applySettings(newSettings); - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); + Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, "true").build(); FeatureFlags.initializeFeatureFlags(nodeSettings); remoteClusterStateService = new RemoteClusterStateService( @@ -2935,11 +2931,10 @@ private void initializeRoutingTable() { .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put(REMOTE_PUBLICATION_SETTING_KEY, "true") .build(); clusterSettings.applySettings(newSettings); - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, @@ -2966,11 +2961,10 @@ private void initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClust .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), mode.name()) + .put(REMOTE_PUBLICATION_SETTING_KEY, true) .build(); clusterSettings.applySettings(newSettings); - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, "true").build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, From 6f45a74765ecc77fcaed85721f6067a292e21162 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 6 Sep 2024 07:28:29 +0530 Subject: [PATCH 074/357] Mute flaky test testRemotePublicationDisabledByRollingRestart (#15772) Signed-off-by: Shivansh Arora --- .../org/opensearch/gateway/remote/RemoteStatePublicationIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 7e7d3060ec7a9..f43d16f7aef43 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -231,6 +231,7 @@ public void testRemotePublicationDownloadStats() { assertDataNodeDownloadStats(nodesStatsResponseDataNode); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15767") public void testRemotePublicationDisabledByRollingRestart() throws Exception { prepareCluster(3, 2, INDEX_NAME, 1, 2); ensureStableCluster(5); From debd04008b56f9f7e54f1d9380dff869d6d81141 Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Fri, 6 Sep 2024 08:19:48 +0530 Subject: [PATCH 075/357] [Snapshot V2] Support pinned timestamp in clone flow (#15524) Signed-off-by: Anshu Agarwal --- .../opensearch/snapshots/CloneSnapshotIT.java | 15 +- .../snapshots/CloneSnapshotV2IT.java | 438 ++++++++++++++++++ .../clone/TransportCloneSnapshotAction.java | 2 +- .../snapshots/SnapshotsService.java | 272 ++++++++++- 4 files changed, 713 insertions(+), 14 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 83f93ab9ff6b5..ddbe0520f0c1f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -64,7 +64,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; @@ -127,7 +126,7 @@ public void testShardClone() throws Exception { } public void testCloneSnapshotIndex() throws Exception { - internalCluster().startClusterManagerOnlyNode(); + internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); internalCluster().startDataOnlyNode(); final String repoName = "repo-name"; createRepository(repoName, "fs"); @@ -318,7 +317,7 @@ public void testClonePreventsSnapshotDelete() throws Exception { indexRandomDocs(indexName, randomIntBetween(20, 100)); final String targetSnapshot = "target-snapshot"; - blockNodeOnAnyFiles(repoName, clusterManagerName); + blockClusterManagerOnWriteIndexFile(repoName); final ActionFuture cloneFuture = startClone(repoName, sourceSnapshot, targetSnapshot, indexName); waitForBlock(clusterManagerName, repoName, TimeValue.timeValueSeconds(30L)); assertFalse(cloneFuture.isDone()); @@ -426,7 +425,7 @@ public void testLongRunningSnapshotAllowsConcurrentClone() throws Exception { } public void testDeletePreventsClone() throws Exception { - final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); internalCluster().startDataOnlyNode(); final String repoName = "repo-name"; createRepository(repoName, "mock"); @@ -439,7 +438,7 @@ public void testDeletePreventsClone() throws Exception { indexRandomDocs(indexName, randomIntBetween(20, 100)); final String targetSnapshot = "target-snapshot"; - blockNodeOnAnyFiles(repoName, clusterManagerName); + blockClusterManagerOnWriteIndexFile(repoName); final ActionFuture deleteFuture = startDeleteSnapshot(repoName, sourceSnapshot); waitForBlock(clusterManagerName, repoName, TimeValue.timeValueSeconds(30L)); assertFalse(deleteFuture.isDone()); @@ -591,7 +590,7 @@ public void testClusterManagerFailoverDuringCloneStep2() throws Exception { public void testExceptionDuringShardClone() throws Exception { // large snapshot pool so blocked snapshot threads from cloning don't prevent concurrent snapshot finalizations - internalCluster().startClusterManagerOnlyNodes(3, LARGE_SNAPSHOT_POOL_SETTINGS); + internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); internalCluster().startDataOnlyNode(); final String repoName = "test-repo"; createRepository(repoName, "mock"); @@ -602,7 +601,7 @@ public void testExceptionDuringShardClone() throws Exception { createFullSnapshot(repoName, sourceSnapshot); final String targetSnapshot = "target-snapshot"; - blockClusterManagerFromFinalizingSnapshotOnSnapFile(repoName); + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); final ActionFuture cloneFuture = startCloneFromDataNode(repoName, sourceSnapshot, targetSnapshot, testIndex); awaitNumberOfSnapshotsInProgress(1); final String clusterManagerNode = internalCluster().getClusterManagerName(); @@ -680,7 +679,7 @@ public void testStartSnapshotWithSuccessfulShardClonePendingFinalization() throw } public void testStartCloneWithSuccessfulShardClonePendingFinalization() throws Exception { - final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); internalCluster().startDataOnlyNode(); final String repoName = "test-repo"; createRepository(repoName, "mock"); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java new file mode 100644 index 0000000000000..c6744ae62db60 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java @@ -0,0 +1,438 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.ActionRunnable; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; + +import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class CloneSnapshotV2IT extends AbstractSnapshotIntegTestCase { + + public void testCloneShallowCopyV2() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-clone-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .get(); + SnapshotInfo sourceSnapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(sourceSnapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(sourceSnapshotInfo.successfulShards(), greaterThan(0)); + assertThat(sourceSnapshotInfo.successfulShards(), equalTo(sourceSnapshotInfo.totalShards())); + assertThat(sourceSnapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + + // Validate that the snapshot was created + final BlobStoreRepository repository = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( + RepositoriesService.class + ).repository(snapshotRepoName); + PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFuture); + + RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); + + assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfo.snapshotId())); + + createIndex(indexName3, getRemoteStoreBackedIndexSettings()); + indexRandomDocs(indexName3, 10); + ensureGreen(indexName3); + + AcknowledgedResponse response = client().admin() + .cluster() + .prepareCloneSnapshot(snapshotRepoName, snapshotName1, "test_clone_snapshot1") + .setIndices("*") + .get(); + assertTrue(response.isAcknowledged()); + awaitClusterManagerFinishRepoOperations(); + + // Validate that snapshot is present in repository data + PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFutureClone); + + repositoryData = repositoryDataPlainActionFutureClone.get(); + assertEquals(repositoryData.getSnapshotIds().size(), 2); + boolean foundCloneInRepoData = false; + SnapshotId cloneSnapshotId = null; + for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { + if (snapshotId.getName().equals("test_clone_snapshot1")) { + foundCloneInRepoData = true; + cloneSnapshotId = snapshotId; + } + } + final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; + SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( + f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) + ); + + assertTrue(foundCloneInRepoData); + + assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); + for (String index : sourceSnapshotInfo.indices()) { + assertTrue(cloneSnapshotInfo.indices().contains(index)); + + } + assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); + } + + public void testCloneShallowCopyAfterDisablingV2() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-clone-snapshot-repo"; + String sourceSnapshotV2 = "test-source-snapshot-v2"; + String sourceSnapshotV1 = "test-source-snapshot-v1"; + String cloneSnapshotV2 = "test-clone-snapshot-v2"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + // create source snapshot which is v2 + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, sourceSnapshotV2) + .setWaitForCompletion(true) + .get(); + SnapshotInfo sourceSnapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(sourceSnapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(sourceSnapshotInfo.successfulShards(), greaterThan(0)); + assertThat(sourceSnapshotInfo.successfulShards(), equalTo(sourceSnapshotInfo.totalShards())); + assertThat(sourceSnapshotInfo.snapshotId().getName(), equalTo(sourceSnapshotV2)); + + // Validate that the snapshot was created + final BlobStoreRepository repository = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( + RepositoriesService.class + ).repository(snapshotRepoName); + PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFuture); + + RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); + + assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfo.snapshotId())); + + createIndex(indexName3, getRemoteStoreBackedIndexSettings()); + indexRandomDocs(indexName3, 10); + ensureGreen(indexName3); + + // disable snapshot v2 in repo + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false) + ) + ); + + // validate that the created snapshot is v1 + CreateSnapshotResponse createSnapshotResponseV1 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, sourceSnapshotV1) + .setWaitForCompletion(true) + .get(); + SnapshotInfo sourceSnapshotInfoV1 = createSnapshotResponseV1.getSnapshotInfo(); + assertThat(sourceSnapshotInfoV1.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(sourceSnapshotInfoV1.successfulShards(), greaterThan(0)); + assertThat(sourceSnapshotInfoV1.successfulShards(), equalTo(sourceSnapshotInfoV1.totalShards())); + assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L)); + + // Validate that snapshot is present in repository data + PlainActionFuture repositoryDataV1PlainActionFuture = new PlainActionFuture<>(); + BlobStoreRepository repositoryV1 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( + RepositoriesService.class + ).repository(snapshotRepoName); + repositoryV1.getRepositoryData(repositoryDataV1PlainActionFuture); + + repositoryData = repositoryDataV1PlainActionFuture.get(); + + assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfoV1.snapshotId())); + assertEquals(repositoryData.getSnapshotIds().size(), 2); + + // clone should get created for v2 snapshot + AcknowledgedResponse response = client().admin() + .cluster() + .prepareCloneSnapshot(snapshotRepoName, sourceSnapshotV2, cloneSnapshotV2) + .setIndices("*") + .get(); + assertTrue(response.isAcknowledged()); + awaitClusterManagerFinishRepoOperations(); + + // Validate that snapshot is present in repository data + PlainActionFuture repositoryDataCloneV2PlainActionFuture = new PlainActionFuture<>(); + BlobStoreRepository repositoryCloneV2 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( + RepositoriesService.class + ).repository(snapshotRepoName); + repositoryCloneV2.getRepositoryData(repositoryDataCloneV2PlainActionFuture); + + repositoryData = repositoryDataCloneV2PlainActionFuture.get(); + + assertEquals(repositoryData.getSnapshotIds().size(), 3); + boolean foundCloneInRepoData = false; + SnapshotId cloneSnapshotId = null; + for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { + if (snapshotId.getName().equals(cloneSnapshotV2)) { + foundCloneInRepoData = true; + cloneSnapshotId = snapshotId; + } + } + final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; + SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( + f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) + ); + + assertTrue(foundCloneInRepoData); + // pinned timestamp value in clone snapshot v2 matches source snapshot v2 + assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); + for (String index : sourceSnapshotInfo.indices()) { + assertTrue(cloneSnapshotInfo.indices().contains(index)); + + } + assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); + + } + + public void testRestoreFromClone() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + + String snapshotRepoName = "test-clone-snapshot-repo"; + String sourceSnapshot = "test-source-snapshot"; + String cloneSnapshot = "test-clone-snapshot"; + + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + String restoredIndexName1 = indexName1 + "-restored"; + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + logger.info("--> create source snapshot"); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, sourceSnapshot) + .setWaitForCompletion(true) + .get(); + SnapshotInfo sourceSnapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(sourceSnapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(sourceSnapshotInfo.successfulShards(), greaterThan(0)); + assertThat(sourceSnapshotInfo.successfulShards(), equalTo(sourceSnapshotInfo.totalShards())); + assertThat(sourceSnapshotInfo.snapshotId().getName(), equalTo(sourceSnapshot)); + + AcknowledgedResponse response = client().admin() + .cluster() + .prepareCloneSnapshot(snapshotRepoName, sourceSnapshot, cloneSnapshot) + .setIndices("*") + .get(); + assertTrue(response.isAcknowledged()); + + DeleteResponse deleteResponse = client().prepareDelete(indexName1, "0").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + ensureGreen(indexName1); + + deleteResponse = client().prepareDelete(indexName1, "1").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + ensureGreen(indexName1); + + // delete the source snapshot + AcknowledgedResponse deleteSnapshotResponse = internalCluster().clusterManagerClient() + .admin() + .cluster() + .prepareDeleteSnapshot(snapshotRepoName, sourceSnapshot) + .get(); + assertAcked(deleteSnapshotResponse); + + deleteResponse = client().prepareDelete(indexName1, "2").execute().actionGet(); + assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED); + ensureGreen(indexName1); + ensureGreen(indexName1); + + // restore from clone + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, cloneSnapshot) + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenameReplacement(restoredIndexName1) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.OK); + ensureGreen(restoredIndexName1, indexName2); + assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1); + assertDocsPresentInIndex(client, indexName2, numDocsInIndex2); + } + + private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { + for (int i = 0; i < numOfDocs; i++) { + String id = Integer.toString(i); + logger.info("checking for index " + indexName + " with docId" + id); + assertTrue("doc with id" + id + " is not present for index " + indexName, client.prepareGet(indexName, id).get().isExists()); + } + } + + private Settings snapshotV2Settings(Path remoteStoreRepoPath) { + String REMOTE_REPO_NAME = "remote-store-repo-name"; + Settings settings = Settings.builder() + .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + return settings; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index 54ef372609390..6da26d9a2da45 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -95,7 +95,7 @@ protected void clusterManagerOperation( ClusterState state, ActionListener listener ) { - snapshotsService.cloneSnapshot(request, ActionListener.map(listener, v -> new AcknowledgedResponse(true))); + snapshotsService.executeClone(request, ActionListener.map(listener, v -> new AcknowledgedResponse(true))); } @Override diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index a2364d96f2098..f6e550525a3e5 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -631,6 +631,34 @@ public void onFailure(Exception e) { ); } + private void cloneSnapshotPinnedTimestamp( + RepositoryData repositoryData, + SnapshotId sourceSnapshot, + Snapshot snapshot, + long timestampToPin, + ActionListener listener + ) { + remoteStorePinnedTimestampService.cloneTimestamp( + timestampToPin, + snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + sourceSnapshot.getUUID(), + snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(), + new ActionListener() { + @Override + public void onResponse(Void unused) { + logger.debug("Timestamp pinned successfully for clone snapshot {}", snapshot.getSnapshotId().getName()); + listener.onResponse(repositoryData); + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to pin timestamp for clone snapshot {} with exception {}", snapshot.getSnapshotId().getName(), e); + listener.onFailure(e); + + } + } + ); + } + private static void ensureSnapshotNameNotRunning( List runningSnapshots, String repositoryName, @@ -652,9 +680,14 @@ private static Map getInFlightIndexIds(List listener) { + /** + * This method does some pre-validation, checks for the presence of source snapshot in repository data. + * For shallow snapshot v2 clone, it checks the pinned timestamp to be greater than zero in the source snapshot. + * + * @param request snapshot request + * @param listener snapshot completion listener + */ + public void executeClone(CloneSnapshotRequest request, ActionListener listener) { final String repositoryName = request.repository(); Repository repository = repositoriesService.repository(repositoryName); if (repository.isReadOnly()) { @@ -663,10 +696,230 @@ public void cloneSnapshot(CloneSnapshotRequest request, ActionListener lis } final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.target()); validate(repositoryName, snapshotName); - // TODO: create snapshot UUID in CloneSnapshotRequest and make this operation idempotent to cleanly deal with transport layer - // retries final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); final Snapshot snapshot = new Snapshot(repositoryName, snapshotId); + try { + final StepListener repositoryDataListener = new StepListener<>(); + repositoriesService.getRepositoryData(repositoryName, repositoryDataListener); + repositoryDataListener.whenComplete(repositoryData -> { + final SnapshotId sourceSnapshotId = repositoryData.getSnapshotIds() + .stream() + .filter(src -> src.getName().equals(request.source())) + .findAny() + .orElseThrow(() -> new SnapshotMissingException(repositoryName, request.source())); + final StepListener snapshotInfoListener = new StepListener<>(); + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + + executor.execute(ActionRunnable.supply(snapshotInfoListener, () -> repository.getSnapshotInfo(sourceSnapshotId))); + + snapshotInfoListener.whenComplete(sourceSnapshotInfo -> { + if (sourceSnapshotInfo.getPinnedTimestamp() > 0) { + if (hasWildCardPatterForCloneSnapshotV2(request.indices()) == false) { + throw new SnapshotException( + repositoryName, + snapshotName, + "Aborting clone for Snapshot-v2, only wildcard pattern '*' is supported for indices" + ); + } + cloneSnapshotV2(request, snapshot, repositoryName, repository, listener); + } else { + cloneSnapshot(request, snapshot, repositoryName, repository, listener); + } + }, e -> listener.onFailure(e)); + }, e -> listener.onFailure(e)); + + } catch (Exception e) { + assert false : new AssertionError(e); + logger.error("Snapshot {} clone failed with exception {}", snapshot.getSnapshotId().getName(), e); + listener.onFailure(e); + } + } + + /** + * This method is responsible for creating a clone of the shallow snapshot v2. + * It pins the same timestamp that is pinned by the source snapshot. + * + * Unlike traditional snapshot operations, this method performs a synchronous clone execution and doesn't + * upload any shard metadata to the snapshot repository. + * The pinned timestamp is later reconciled with remote store segment and translog metadata files during the restore + * operation. + * + * @param request snapshot request + * @param snapshot clone snapshot + * @param repositoryName snapshot repository name + * @param repository snapshot repository + * @param listener completion listener + */ + public void cloneSnapshotV2( + CloneSnapshotRequest request, + Snapshot snapshot, + String repositoryName, + Repository repository, + ActionListener listener + ) { + + long startTime = System.currentTimeMillis(); + ClusterState currentState = clusterService.state(); + String snapshotName = snapshot.getSnapshotId().getName(); + repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) { + private SnapshotsInProgress.Entry newEntry; + private SnapshotId sourceSnapshotId; + private List indicesForSnapshot; + + @Override + public ClusterState execute(ClusterState currentState) { + createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final List runningSnapshots = snapshots.entries(); + + sourceSnapshotId = repositoryData.getSnapshotIds() + .stream() + .filter(src -> src.getName().equals(request.source())) + .findAny() + .orElseThrow(() -> new SnapshotMissingException(repositoryName, request.source())); + + final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom( + SnapshotDeletionsInProgress.TYPE, + SnapshotDeletionsInProgress.EMPTY + ); + if (deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshots().contains(sourceSnapshotId))) { + throw new ConcurrentSnapshotExecutionException( + repositoryName, + sourceSnapshotId.getName(), + "cannot clone from snapshot that is being deleted" + ); + } + indicesForSnapshot = new ArrayList<>(); + for (IndexId indexId : repositoryData.getIndices().values()) { + if (repositoryData.getSnapshots(indexId).contains(sourceSnapshotId)) { + indicesForSnapshot.add(indexId.getName()); + } + } + + newEntry = SnapshotsInProgress.startClone( + snapshot, + sourceSnapshotId, + repositoryData.resolveIndices(indicesForSnapshot), + threadPool.absoluteTimeInMillis(), + repositoryData.getGenId(), + minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null) + ); + final List newEntries = new ArrayList<>(runningSnapshots); + newEntries.add(newEntry); + return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(newEntries)).build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to clone snapshot-v2", repositoryName, snapshotName), e); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) { + logger.info("snapshot-v2 clone [{}] started", snapshot); + final StepListener snapshotInfoListener = new StepListener<>(); + final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + + executor.execute(ActionRunnable.supply(snapshotInfoListener, () -> repository.getSnapshotInfo(sourceSnapshotId))); + final ShardGenerations shardGenerations = repositoryData.shardGenerations(); + + snapshotInfoListener.whenComplete(snapshotInfo -> { + final SnapshotInfo cloneSnapshotInfo = new SnapshotInfo( + snapshot.getSnapshotId(), + indicesForSnapshot, + newEntry.dataStreams(), + startTime, + null, + System.currentTimeMillis(), + snapshotInfo.totalShards(), + Collections.emptyList(), + newEntry.includeGlobalState(), + newEntry.userMetadata(), + true, + snapshotInfo.getPinnedTimestamp() + ); + if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2 clone, no longer cluster manager"); + } + final StepListener pinnedTimestampListener = new StepListener<>(); + pinnedTimestampListener.whenComplete(repoData -> { + logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); + listener.onResponse(null); + }, listener::onFailure); + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot( + currentState.metadata(), + newEntry.includeGlobalState(), + false, + newEntry.dataStreams(), + newEntry.indices() + ), + cloneSnapshotInfo, + repositoryData.getVersion(sourceSnapshotId), + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException( + repositoryName, + snapshotName, + "Aborting Snapshot-v2 clone, no longer cluster manager" + ) + ); + return; + } + cloneSnapshotPinnedTimestamp( + repositoryData, + sourceSnapshotId, + snapshot, + snapshotInfo.getPinnedTimestamp(), + pinnedTimestampListener + ); + } + + @Override + public void onFailure(Exception e) { + logger.error( + "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ", + repositoryName, + snapshotName + ); + listener.onFailure(e); + } + } + ); + + }, listener::onFailure); + } + + @Override + public TimeValue timeout() { + return request.clusterManagerNodeTimeout(); + } + }, "clone_snapshot_v2 [" + request.source() + "][" + snapshotName + ']', listener::onFailure); + } + + // TODO: It is worth revisiting the design choice of creating a placeholder entry in snapshots-in-progress here once we have a cache + // for repository metadata and loading it has predictable performance + public void cloneSnapshot( + CloneSnapshotRequest request, + Snapshot snapshot, + String repositoryName, + Repository repository, + ActionListener listener + ) { + String snapshotName = snapshot.getSnapshotId().getName(); + initializingClones.add(snapshot); repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask() { @@ -3499,6 +3752,15 @@ private void startExecutableClones(SnapshotsInProgress snapshotsInProgress, @Nul } } + private boolean hasWildCardPatterForCloneSnapshotV2(String[] indices) { + for (String index : indices) { + if ("*".equals(index)) { + return true; + } + } + return false; + } + private class UpdateSnapshotStatusAction extends TransportClusterManagerNodeAction< UpdateIndexShardSnapshotStatusRequest, UpdateIndexShardSnapshotStatusResponse> { From 3c6019dfca1e89bde6d82eb7d366621245defb86 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 6 Sep 2024 11:02:59 +0530 Subject: [PATCH 076/357] Add wait in replica recovery for allocation id to propagate on source node (#15558) * Add wait for target allocation id to appear Signed-off-by: Gaurav Bafna * making waitForAssignment same Signed-off-by: Gaurav Bafna * Add more test Signed-off-by: Gaurav Bafna --------- Signed-off-by: Gaurav Bafna --- .../LocalStorePeerRecoverySourceHandler.java | 19 +- .../recovery/RecoverySourceHandler.java | 51 +++++ .../RemoteStorePeerRecoverySourceHandler.java | 6 +- ...alStorePeerRecoverySourceHandlerTests.java | 204 ++++++++++++++++++ ...teStorePeerRecoverySourceHandlerTests.java | 162 ++++++++++++++ 5 files changed, 423 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java index ac6b2e6b77d18..234e9ba66f340 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandler.java @@ -12,15 +12,12 @@ import org.opensearch.action.StepListener; import org.opensearch.action.support.ThreadedActionListener; import org.opensearch.action.support.replication.ReplicationResponse; -import org.opensearch.cluster.routing.IndexShardRoutingTable; -import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.SetOnce; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.index.engine.RecoveryEngineException; -import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.RetentionLeaseNotFoundException; import org.opensearch.index.seqno.RetentionLeases; @@ -58,21 +55,7 @@ public LocalStorePeerRecoverySourceHandler( @Override protected void innerRecoveryToTarget(ActionListener listener, Consumer onFailure) throws IOException { final SetOnce retentionLeaseRef = new SetOnce<>(); - - RunUnderPrimaryPermit.run(() -> { - final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable(); - ShardRouting targetShardRouting = routingTable.getByAllocationId(request.targetAllocationId()); - if (targetShardRouting == null) { - logger.debug( - "delaying recovery of {} as it is not listed as assigned to target node {}", - request.shardId(), - request.targetNode() - ); - throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); - } - assert targetShardRouting.initializing() : "expected recovery target to be initializing but was " + targetShardRouting; - retentionLeaseRef.set(shard.getRetentionLeases().get(ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting))); - }, shardId + " validating recovery target [" + request.targetAllocationId() + "] registered ", shard, cancellableThreads, logger); + waitForAssignmentPropagate(retentionLeaseRef); final Closeable retentionLock = shard.acquireHistoryRetentionLock(); resources.add(retentionLock); final long startingSeqNo; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 8966516d3ef7f..7aa90d2f9fa3e 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -41,10 +41,14 @@ import org.apache.lucene.util.ArrayUtil; import org.opensearch.action.ActionRunnable; import org.opensearch.action.StepListener; +import org.opensearch.action.bulk.BackoffPolicy; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.ThreadedActionListener; import org.opensearch.action.support.replication.ReplicationResponse; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.SetOnce; import org.opensearch.common.StopWatch; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; @@ -59,6 +63,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.engine.RecoveryEngineException; +import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.RetentionLeaseNotFoundException; import org.opensearch.index.seqno.RetentionLeases; @@ -79,12 +84,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.IntSupplier; import java.util.stream.StreamSupport; @@ -191,6 +198,50 @@ public void recoverToTarget(ActionListener listener) { protected abstract void innerRecoveryToTarget(ActionListener listener, Consumer onFailure) throws IOException; + /* + Waits for cluster state propagation of assignment of replica on the target node + */ + void waitForAssignmentPropagate(SetOnce retentionLeaseRef) { + BackoffPolicy EXPONENTIAL_BACKOFF_POLICY = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(1000), 5); + AtomicReference targetShardRouting = new AtomicReference<>(); + Iterator backoffDelayIterator = EXPONENTIAL_BACKOFF_POLICY.iterator(); + while (backoffDelayIterator.hasNext()) { + RunUnderPrimaryPermit.run(() -> { + final IndexShardRoutingTable routingTable = shard.getReplicationGroup().getRoutingTable(); + targetShardRouting.set(routingTable.getByAllocationId(request.targetAllocationId())); + if (targetShardRouting.get() == null) { + logger.info( + "delaying recovery of {} as it is not listed as assigned to target node {}", + request.shardId(), + request.targetNode() + ); + Thread.sleep(backoffDelayIterator.next().millis()); + } + if (targetShardRouting.get() != null) { + assert targetShardRouting.get().initializing() : "expected recovery target to be initializing but was " + + targetShardRouting; + retentionLeaseRef.set( + shard.getRetentionLeases().get(ReplicationTracker.getPeerRecoveryRetentionLeaseId(targetShardRouting.get())) + ); + } + + }, + shardId + " validating recovery target [" + request.targetAllocationId() + "] registered ", + shard, + cancellableThreads, + logger + ); + + if (targetShardRouting.get() != null) { + return; + } + } + if (targetShardRouting.get() != null) { + return; + } + throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); + } + protected void finalizeStepAndCompleteFuture( long startingSeqNo, StepListener sendSnapshotStep, diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java index 66c7a3b48f28f..383ed92314165 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandler.java @@ -10,11 +10,13 @@ import org.apache.lucene.index.IndexCommit; import org.opensearch.action.StepListener; +import org.opensearch.common.SetOnce; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lease.Releasable; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.index.engine.RecoveryEngineException; +import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.RunUnderPrimaryPermit; @@ -48,7 +50,8 @@ protected void innerRecoveryToTarget(ActionListener listener, // A replica of an index with remote translog does not require the translogs locally and keeps receiving the // updated segments file on refresh, flushes, and merges. In recovery, here, only file-based recovery is performed // and there is no translog replay done. - + final SetOnce retentionLeaseRef = new SetOnce<>(); + waitForAssignmentPropagate(retentionLeaseRef); final StepListener sendFileStep = new StepListener<>(); final StepListener prepareEngineStep = new StepListener<>(); final StepListener sendSnapshotStep = new StepListener<>(); @@ -102,4 +105,5 @@ protected void innerRecoveryToTarget(ActionListener listener, finalizeStepAndCompleteFuture(startingSeqNo, sendSnapshotStep, sendFileStep, prepareEngineStep, onFailure); } + } diff --git a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java index 4685a7196b85a..a4598c4294a33 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/LocalStorePeerRecoverySourceHandlerTests.java @@ -52,6 +52,8 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Numbers; import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; @@ -141,6 +143,8 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -746,6 +750,206 @@ void phase2( assertFalse(phase2Called.get()); } + /* + If the replica allocation id is not reflected in source nodes routing table even after retries, + recoveries should fail + */ + public void testThrowExceptionOnNoTargetInRouting() throws IOException { + final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service); + final StartRecoveryRequest request = getStartRecoveryRequest(); + final IndexShard shard = mock(IndexShard.class); + when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); + when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class)); + when(shard.isRelocatedPrimary()).thenReturn(false); + final org.opensearch.index.shard.ReplicationGroup replicationGroup = mock(org.opensearch.index.shard.ReplicationGroup.class); + final IndexShardRoutingTable routingTable = mock(IndexShardRoutingTable.class); + when(routingTable.getByAllocationId(anyString())).thenReturn(null); + when(shard.getReplicationGroup()).thenReturn(replicationGroup); + when(replicationGroup.getRoutingTable()).thenReturn(routingTable); + when(shard.acquireSafeIndexCommit()).thenReturn(mock(GatedCloseable.class)); + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[0]).onResponse(() -> {}); + return null; + }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), any()); + + final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)) + .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())) + ); + if (randomBoolean()) { + indexMetadata.state(IndexMetadata.State.CLOSE); + } + when(shard.indexSettings()).thenReturn(new IndexSettings(indexMetadata.build(), Settings.EMPTY)); + + final AtomicBoolean phase1Called = new AtomicBoolean(); + final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean(); + final AtomicBoolean phase2Called = new AtomicBoolean(); + final RecoverySourceHandler handler = new LocalStorePeerRecoverySourceHandler( + shard, + mock(RecoveryTargetHandler.class), + threadPool, + request, + Math.toIntExact(recoverySettings.getChunkSize().getBytes()), + between(1, 8), + between(1, 8) + ) { + + @Override + void phase1( + IndexCommit snapshot, + long startingSeqNo, + IntSupplier translogOps, + ActionListener listener, + boolean skipCreateRetentionLeaseStep + ) { + phase1Called.set(true); + super.phase1(snapshot, startingSeqNo, translogOps, listener, skipCreateRetentionLeaseStep); + } + + @Override + void prepareTargetForTranslog(int totalTranslogOps, ActionListener listener) { + prepareTargetForTranslogCalled.set(true); + super.prepareTargetForTranslog(totalTranslogOps, listener); + } + + @Override + void phase2( + long startingSeqNo, + long endingSeqNo, + Translog.Snapshot snapshot, + long maxSeenAutoIdTimestamp, + long maxSeqNoOfUpdatesOrDeletes, + RetentionLeases retentionLeases, + long mappingVersion, + ActionListener listener + ) throws IOException { + phase2Called.set(true); + super.phase2( + startingSeqNo, + endingSeqNo, + snapshot, + maxSeenAutoIdTimestamp, + maxSeqNoOfUpdatesOrDeletes, + retentionLeases, + mappingVersion, + listener + ); + } + + }; + PlainActionFuture future = new PlainActionFuture<>(); + expectThrows(DelayRecoveryException.class, () -> { + handler.recoverToTarget(future); + future.actionGet(); + }); + verify(routingTable, times(5)).getByAllocationId(null); + assertFalse(phase1Called.get()); + assertFalse(prepareTargetForTranslogCalled.get()); + assertFalse(phase2Called.get()); + } + + /* + Tests when the replica allocation id is reflected in source nodes routing table even after 1 retry + */ + public void testTargetInRoutingInSecondAttempt() throws IOException { + final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service); + final StartRecoveryRequest request = getStartRecoveryRequest(); + final IndexShard shard = mock(IndexShard.class); + when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); + when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class)); + when(shard.isRelocatedPrimary()).thenReturn(false); + when(shard.getRetentionLeases()).thenReturn(mock(RetentionLeases.class)); + final org.opensearch.index.shard.ReplicationGroup replicationGroup = mock(org.opensearch.index.shard.ReplicationGroup.class); + final IndexShardRoutingTable routingTable = mock(IndexShardRoutingTable.class); + final ShardRouting shardRouting = mock(ShardRouting.class); + when(shardRouting.initializing()).thenReturn(true); + when(shardRouting.currentNodeId()).thenReturn("node"); + when(routingTable.getByAllocationId(any())).thenReturn(null, shardRouting); + when(shard.getReplicationGroup()).thenReturn(replicationGroup); + when(replicationGroup.getRoutingTable()).thenReturn(routingTable); + when(shard.acquireSafeIndexCommit()).thenReturn(mock(GatedCloseable.class)); + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[0]).onResponse(() -> {}); + return null; + }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), any()); + + final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)) + .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())) + ); + if (randomBoolean()) { + indexMetadata.state(IndexMetadata.State.CLOSE); + } + when(shard.indexSettings()).thenReturn(new IndexSettings(indexMetadata.build(), Settings.EMPTY)); + + final AtomicBoolean phase1Called = new AtomicBoolean(); + final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean(); + final AtomicBoolean phase2Called = new AtomicBoolean(); + final RecoverySourceHandler handler = new LocalStorePeerRecoverySourceHandler( + shard, + mock(RecoveryTargetHandler.class), + threadPool, + request, + Math.toIntExact(recoverySettings.getChunkSize().getBytes()), + between(1, 8), + between(1, 8) + ) { + + @Override + void phase1( + IndexCommit snapshot, + long startingSeqNo, + IntSupplier translogOps, + ActionListener listener, + boolean skipCreateRetentionLeaseStep + ) { + phase1Called.set(true); + super.phase1(snapshot, startingSeqNo, translogOps, listener, skipCreateRetentionLeaseStep); + } + + @Override + void prepareTargetForTranslog(int totalTranslogOps, ActionListener listener) { + prepareTargetForTranslogCalled.set(true); + super.prepareTargetForTranslog(totalTranslogOps, listener); + } + + @Override + void phase2( + long startingSeqNo, + long endingSeqNo, + Translog.Snapshot snapshot, + long maxSeenAutoIdTimestamp, + long maxSeqNoOfUpdatesOrDeletes, + RetentionLeases retentionLeases, + long mappingVersion, + ActionListener listener + ) throws IOException { + phase2Called.set(true); + super.phase2( + startingSeqNo, + endingSeqNo, + snapshot, + maxSeenAutoIdTimestamp, + maxSeqNoOfUpdatesOrDeletes, + retentionLeases, + mappingVersion, + listener + ); + } + + }; + handler.waitForAssignmentPropagate(new SetOnce<>()); + verify(routingTable, times(2)).getByAllocationId(null); + } + public void testCancellationsDoesNotLeakPrimaryPermits() throws Exception { final CancellableThreads cancellableThreads = new CancellableThreads(); final IndexShard shard = mock(IndexShard.class); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 131514eb019b3..7f913f3c8596a 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -8,20 +8,64 @@ package org.opensearch.indices.recovery; +import org.apache.lucene.index.IndexCommit; +import org.opensearch.Version; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.UUIDs; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.index.seqno.ReplicationTracker; +import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.seqno.SeqNoStats; +import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.store.Store; +import org.opensearch.index.translog.Translog; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.IndexSettingsModule; +import org.opensearch.test.VersionUtils; +import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.IntSupplier; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLevelReplicationTestCase { + private static final IndexSettings INDEX_SETTINGS = IndexSettingsModule.newIndexSettings( + "index", + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build() + ); + private final ShardId shardId = new ShardId(INDEX_SETTINGS.getIndex(), 1); + + private final ClusterSettings service = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") @@ -73,4 +117,122 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { assertFalse(primary.getRetentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replica2.routingEntry()))); } } + + public void testThrowExceptionOnNoTargetInRouting() throws IOException { + final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service); + final StartRecoveryRequest request = getStartRecoveryRequest(); + final IndexShard shard = mock(IndexShard.class); + when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); + when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class)); + when(shard.isRelocatedPrimary()).thenReturn(false); + final org.opensearch.index.shard.ReplicationGroup replicationGroup = mock(org.opensearch.index.shard.ReplicationGroup.class); + final IndexShardRoutingTable routingTable = mock(IndexShardRoutingTable.class); + when(routingTable.getByAllocationId(anyString())).thenReturn(null); + when(shard.getReplicationGroup()).thenReturn(replicationGroup); + when(replicationGroup.getRoutingTable()).thenReturn(routingTable); + when(shard.acquireSafeIndexCommit()).thenReturn(mock(GatedCloseable.class)); + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[0]).onResponse(() -> {}); + return null; + }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), any()); + + final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)) + .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())) + ); + if (randomBoolean()) { + indexMetadata.state(IndexMetadata.State.CLOSE); + } + when(shard.indexSettings()).thenReturn(new IndexSettings(indexMetadata.build(), Settings.EMPTY)); + + final AtomicBoolean phase1Called = new AtomicBoolean(); + final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean(); + final AtomicBoolean phase2Called = new AtomicBoolean(); + final RecoverySourceHandler handler = new RemoteStorePeerRecoverySourceHandler( + shard, + mock(RecoveryTargetHandler.class), + threadPool, + request, + Math.toIntExact(recoverySettings.getChunkSize().getBytes()), + between(1, 8), + between(1, 8) + ) { + + @Override + void phase1( + IndexCommit snapshot, + long startingSeqNo, + IntSupplier translogOps, + ActionListener listener, + boolean skipCreateRetentionLeaseStep + ) { + phase1Called.set(true); + super.phase1(snapshot, startingSeqNo, translogOps, listener, skipCreateRetentionLeaseStep); + } + + @Override + void prepareTargetForTranslog(int totalTranslogOps, ActionListener listener) { + prepareTargetForTranslogCalled.set(true); + super.prepareTargetForTranslog(totalTranslogOps, listener); + } + + @Override + void phase2( + long startingSeqNo, + long endingSeqNo, + Translog.Snapshot snapshot, + long maxSeenAutoIdTimestamp, + long maxSeqNoOfUpdatesOrDeletes, + RetentionLeases retentionLeases, + long mappingVersion, + ActionListener listener + ) throws IOException { + phase2Called.set(true); + super.phase2( + startingSeqNo, + endingSeqNo, + snapshot, + maxSeenAutoIdTimestamp, + maxSeqNoOfUpdatesOrDeletes, + retentionLeases, + mappingVersion, + listener + ); + } + + }; + PlainActionFuture future = new PlainActionFuture<>(); + expectThrows(DelayRecoveryException.class, () -> { + handler.recoverToTarget(future); + future.actionGet(); + }); + verify(routingTable, times(5)).getByAllocationId(null); + assertFalse(phase1Called.get()); + assertFalse(prepareTargetForTranslogCalled.get()); + assertFalse(phase2Called.get()); + } + + public StartRecoveryRequest getStartRecoveryRequest() throws IOException { + Store.MetadataSnapshot metadataSnapshot = randomBoolean() + ? Store.MetadataSnapshot.EMPTY + : new Store.MetadataSnapshot( + Collections.emptyMap(), + Collections.singletonMap(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()), + randomIntBetween(0, 100) + ); + return new StartRecoveryRequest( + shardId, + null, + new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), + new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), + metadataSnapshot, + randomBoolean(), + randomNonNegativeLong(), + randomBoolean() || metadataSnapshot.getHistoryUUID() == null ? SequenceNumbers.UNASSIGNED_SEQ_NO : randomNonNegativeLong() + ); + } } From cbdcbb7da3d87fdccfca164ac8f6914a55dc19cb Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 6 Sep 2024 14:38:21 +0530 Subject: [PATCH 077/357] [Remote State] Upload incremental cluster state on master re-election (#15145) * Upload incremental cluster state on master re-election Signed-off-by: Shivansh Arora --- CHANGELOG.md | 1 + .../remote/RemoteStatePublicationIT.java | 60 ++- .../remotestore/BaseRemoteStoreRestoreIT.java | 25 -- .../RemoteStoreBaseIntegTestCase.java | 17 +- .../coordination/CoordinationState.java | 72 +++- .../PublicationTransportHandler.java | 17 +- .../RemoteStatePublishRequest.java | 51 +++ .../opensearch/gateway/GatewayMetaState.java | 54 ++- .../remote/RemoteClusterStateService.java | 44 +- .../coordination/CoordinationStateTests.java | 386 ++++++++++++++++-- .../GatewayMetaStatePersistedStateTests.java | 123 +++++- .../RemoteClusterStateServiceTests.java | 8 +- 12 files changed, 732 insertions(+), 126 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/coordination/RemoteStatePublishRequest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 26358d269a2ea..deb9b7d0bcad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) +- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) ### Dependencies diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index f43d16f7aef43..faab3645ae894 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -20,6 +20,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryStats; +import org.opensearch.gateway.GatewayMetaState; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore; @@ -43,6 +44,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; @@ -74,7 +76,7 @@ public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase { private static final String REMOTE_STATE_PREFIX = "!"; private static final String REMOTE_ROUTING_PREFIX = "_"; private boolean isRemoteStateEnabled = true; - private String isRemotePublicationEnabled = "true"; + private boolean isRemotePublicationEnabled = true; private boolean hasRemoteStateCharPrefix; private boolean hasRemoteRoutingCharPrefix; @@ -82,7 +84,7 @@ public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase { public void setup() { asyncUploadMockFsRepo = false; isRemoteStateEnabled = true; - isRemotePublicationEnabled = "true"; + isRemotePublicationEnabled = true; hasRemoteStateCharPrefix = randomBoolean(); hasRemoteRoutingCharPrefix = randomBoolean(); } @@ -112,6 +114,7 @@ protected Settings nodeSettings(int nodeOrdinal) { RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(), RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE ) + .put(REMOTE_PUBLICATION_SETTING_KEY, isRemotePublicationEnabled) .put( RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.getKey(), hasRemoteStateCharPrefix ? REMOTE_STATE_PREFIX : "" @@ -341,6 +344,59 @@ public void doAfterNodes(int n, Client client) { }); } + public void testMasterReElectionUsesIncrementalUpload() throws IOException { + prepareCluster(3, 2, INDEX_NAME, 1, 1); + PersistedStateRegistry persistedStateRegistry = internalCluster().getClusterManagerNodeInstance(PersistedStateRegistry.class); + GatewayMetaState.RemotePersistedState remotePersistedState = (GatewayMetaState.RemotePersistedState) persistedStateRegistry + .getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); + ClusterMetadataManifest manifest = remotePersistedState.getLastAcceptedManifest(); + // force elected master to step down + internalCluster().stopCurrentClusterManagerNode(); + ensureStableCluster(4); + + persistedStateRegistry = internalCluster().getClusterManagerNodeInstance(PersistedStateRegistry.class); + CoordinationState.PersistedState persistedStateAfterElection = persistedStateRegistry.getPersistedState( + PersistedStateRegistry.PersistedStateType.REMOTE + ); + ClusterMetadataManifest manifestAfterElection = persistedStateAfterElection.getLastAcceptedManifest(); + + // coordination metadata is updated, it will be unequal + assertNotEquals(manifest.getCoordinationMetadata(), manifestAfterElection.getCoordinationMetadata()); + // all other attributes are not uploaded again and will be pointing to same files in manifest after new master is elected + assertEquals(manifest.getClusterUUID(), manifestAfterElection.getClusterUUID()); + assertEquals(manifest.getIndices(), manifestAfterElection.getIndices()); + assertEquals(manifest.getSettingsMetadata(), manifestAfterElection.getSettingsMetadata()); + assertEquals(manifest.getTemplatesMetadata(), manifestAfterElection.getTemplatesMetadata()); + assertEquals(manifest.getCustomMetadataMap(), manifestAfterElection.getCustomMetadataMap()); + assertEquals(manifest.getRoutingTableVersion(), manifest.getRoutingTableVersion()); + assertEquals(manifest.getIndicesRouting(), manifestAfterElection.getIndicesRouting()); + } + + public void testVotingConfigAreCommitted() throws ExecutionException, InterruptedException { + prepareCluster(3, 2, INDEX_NAME, 1, 2); + ensureStableCluster(5); + ensureGreen(INDEX_NAME); + // add two new nodes to the cluster, to update the voting config + internalCluster().startClusterManagerOnlyNodes(2, Settings.EMPTY); + ensureStableCluster(7); + + internalCluster().getInstances(PersistedStateRegistry.class).forEach(persistedStateRegistry -> { + CoordinationState.PersistedState localState = persistedStateRegistry.getPersistedState( + PersistedStateRegistry.PersistedStateType.LOCAL + ); + CoordinationState.PersistedState remoteState = persistedStateRegistry.getPersistedState( + PersistedStateRegistry.PersistedStateType.REMOTE + ); + if (remoteState != null) { + assertEquals( + localState.getLastAcceptedState().getLastCommittedConfiguration(), + remoteState.getLastAcceptedState().getLastCommittedConfiguration() + ); + assertEquals(5, remoteState.getLastAcceptedState().getLastCommittedConfiguration().getNodeIds().size()); + } + }); + } + private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) { // assert cluster state stats for data node DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats(); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java index 280fd13f0fdcf..e8df2c8686610 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/BaseRemoteStoreRestoreIT.java @@ -76,29 +76,4 @@ protected void verifyRestoredData(Map indexStats, String indexName protected void verifyRestoredData(Map indexStats, String indexName) throws Exception { verifyRestoredData(indexStats, indexName, true); } - - public void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { - prepareCluster(numClusterManagerNodes, numDataOnlyNodes, indices, replicaCount, shardCount, Settings.EMPTY); - } - - public void prepareCluster( - int numClusterManagerNodes, - int numDataOnlyNodes, - String indices, - int replicaCount, - int shardCount, - Settings settings - ) { - prepareCluster(numClusterManagerNodes, numDataOnlyNodes, settings); - for (String index : indices.split(",")) { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); - ensureYellowAndNoInitializingShards(index); - ensureGreen(index); - } - } - - public void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, Settings settings) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes, settings); - internalCluster().startDataOnlyNodes(numDataOnlyNodes, settings); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index ba06bb463e5a8..bcb0d54c0a25c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -351,13 +351,7 @@ protected void restore(boolean restoreAllShards, String... indices) { } protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); - internalCluster().startDataOnlyNodes(numDataOnlyNodes); - for (String index : indices.split(",")) { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); - ensureYellowAndNoInitializingShards(index); - ensureGreen(index); - } + prepareCluster(numClusterManagerNodes, numDataOnlyNodes, indices, replicaCount, shardCount, Settings.EMPTY); } protected void prepareCluster( @@ -368,11 +362,16 @@ protected void prepareCluster( int shardCount, Settings settings ) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes, settings); - internalCluster().startDataOnlyNodes(numDataOnlyNodes, settings); + prepareCluster(numClusterManagerNodes, numDataOnlyNodes, settings); for (String index : indices.split(",")) { createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); + ensureYellowAndNoInitializingShards(index); ensureGreen(index); } } + + protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, Settings settings) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes, settings); + internalCluster().startDataOnlyNodes(numDataOnlyNodes, settings); + } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 9c883175e3ee0..9cffc7051d756 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -40,6 +40,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.gateway.remote.ClusterMetadataManifest; import java.io.Closeable; import java.io.IOException; @@ -104,6 +105,7 @@ public CoordinationState( .getLastAcceptedConfiguration(); this.publishVotes = new VoteCollection(); this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); + // ToDo: revisit this check while making the setting dynamic this.isRemotePublicationEnabled = isRemoteStateEnabled && REMOTE_PUBLICATION_SETTING.get(settings) && localNode.isRemoteStatePublicationEnabled(); @@ -459,6 +461,9 @@ public PublishResponse handlePublishRequest(PublishRequest publishRequest) { clusterState.term() ); persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).setLastAcceptedState(clusterState); + if (shouldUpdateRemotePersistedState(publishRequest)) { + updateRemotePersistedStateOnPublishRequest(publishRequest); + } assert getLastAcceptedState() == clusterState; return new PublishResponse(clusterState.term(), clusterState.version()); @@ -571,6 +576,9 @@ public void handleCommit(ApplyCommitRequest applyCommit) { ); persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).markLastAcceptedStateAsCommitted(); + if (shouldCommitRemotePersistedState()) { + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).markLastAcceptedStateAsCommitted(); + } assert getLastCommittedConfiguration().equals(getLastAcceptedConfiguration()); } @@ -616,6 +624,33 @@ public void close() throws IOException { IOUtils.close(persistedStateRegistry); } + private boolean shouldUpdateRemotePersistedState(PublishRequest publishRequest) { + return persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE) != null + && publishRequest.getAcceptedState().getNodes().isLocalNodeElectedClusterManager() == false; + } + + private void updateRemotePersistedStateOnPublishRequest(PublishRequest publishRequest) { + if (publishRequest instanceof RemoteStatePublishRequest) { + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).setLastAcceptedState(publishRequest.getAcceptedState()); + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE) + .setLastAcceptedManifest(((RemoteStatePublishRequest) publishRequest).getAcceptedManifest()); + } else { + // We will end up here if PublishRequest was sent not using Remote Store even with remote persisted state on this node + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).setLastAcceptedState(null); + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).setLastAcceptedManifest(null); + } + } + + private boolean shouldCommitRemotePersistedState() { + return persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE) != null + && persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL) + .getLastAcceptedState() + .getNodes() + .isLocalNodeElectedClusterManager() == false + && persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState() != null + && persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedManifest() != null; + } + /** * Pluggable persistence layer for {@link CoordinationState}. * @@ -653,6 +688,22 @@ public interface PersistedState extends Closeable { */ PersistedStateStats getStats(); + /** + * Returns the last accepted {@link ClusterMetadataManifest}. + * + * @return The last accepted {@link ClusterMetadataManifest}, or null if no manifest + * has been accepted yet. + */ + default ClusterMetadataManifest getLastAcceptedManifest() { + // return null by default, this method needs to be overridden wherever required + return null; + } + + /** + * Sets the last accepted {@link ClusterMetadataManifest}. + */ + default void setLastAcceptedManifest(ClusterMetadataManifest manifest) {} + /** * Marks the last accepted cluster state as committed. * After a successful call to this method, {@link #getLastAcceptedState()} should return the last cluster state that was set, @@ -661,14 +712,7 @@ public interface PersistedState extends Closeable { */ default void markLastAcceptedStateAsCommitted() { final ClusterState lastAcceptedState = getLastAcceptedState(); - Metadata.Builder metadataBuilder = null; - if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) { - final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder(lastAcceptedState.coordinationMetadata()) - .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration()) - .build(); - metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); - metadataBuilder.coordinationMetadata(coordinationMetadata); - } + Metadata.Builder metadataBuilder = commitVotingConfiguration(lastAcceptedState); // if we receive a commit from a Zen1 cluster-manager that has not recovered its state yet, // the cluster uuid might not been known yet. assert lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false @@ -693,6 +737,18 @@ default void markLastAcceptedStateAsCommitted() { } } + default Metadata.Builder commitVotingConfiguration(ClusterState lastAcceptedState) { + Metadata.Builder metadataBuilder = null; + if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) { + final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder(lastAcceptedState.coordinationMetadata()) + .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration()) + .build(); + metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); + metadataBuilder.coordinationMetadata(coordinationMetadata); + } + return metadataBuilder; + } + default void close() throws IOException {} } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index ca36011b3a0e9..cdf331b7bb577 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -199,7 +199,7 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque } fullClusterStateReceivedCount.incrementAndGet(); logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length()); - final PublishWithJoinResponse response = acceptState(incomingState); + final PublishWithJoinResponse response = acceptState(incomingState, null); lastSeenClusterState.set(incomingState); return response; } else { @@ -230,7 +230,7 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque incomingState.stateUUID(), request.bytes().length() ); - final PublishWithJoinResponse response = acceptState(incomingState); + final PublishWithJoinResponse response = acceptState(incomingState, null); lastSeenClusterState.compareAndSet(lastSeen, incomingState); return response; } @@ -281,7 +281,7 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest true ); fullClusterStateReceivedCount.incrementAndGet(); - final PublishWithJoinResponse response = acceptState(clusterState); + final PublishWithJoinResponse response = acceptState(clusterState, manifest); lastSeenClusterState.set(clusterState); return response; } else { @@ -300,7 +300,7 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest transportService.getLocalNode().getId() ); compatibleClusterStateDiffReceivedCount.incrementAndGet(); - final PublishWithJoinResponse response = acceptState(clusterState); + final PublishWithJoinResponse response = acceptState(clusterState, manifest); lastSeenClusterState.compareAndSet(lastSeen, clusterState); return response; } @@ -314,7 +314,7 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest } } - private PublishWithJoinResponse acceptState(ClusterState incomingState) { + private PublishWithJoinResponse acceptState(ClusterState incomingState, ClusterMetadataManifest manifest) { // if the state is coming from the current node, use original request instead (see currentPublishRequestToSelf for explanation) if (transportService.getLocalNode().equals(incomingState.nodes().getClusterManagerNode())) { final PublishRequest publishRequest = currentPublishRequestToSelf.get(); @@ -324,6 +324,9 @@ private PublishWithJoinResponse acceptState(ClusterState incomingState) { return handlePublishRequest.apply(publishRequest); } } + if (manifest != null) { + return handlePublishRequest.apply(new RemoteStatePublishRequest(incomingState, manifest)); + } return handlePublishRequest.apply(new PublishRequest(incomingState)); } @@ -539,7 +542,7 @@ public String executor() { } public void sendClusterState(DiscoveryNode destination, ActionListener listener) { - logger.info("sending cluster state over transport to node: {}", destination.getName()); + logger.debug("sending cluster state over transport to node: {}", destination.getName()); if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); sendFullClusterState(destination, listener); @@ -639,7 +642,7 @@ public class RemotePublicationContext extends PublicationContext { @Override public void sendClusterState(final DiscoveryNode destination, final ActionListener listener) { try { - logger.info("sending remote cluster state to node: {}", destination.getName()); + logger.debug("sending remote cluster state to node: {}", destination.getName()); final String manifestFileName = ((RemotePersistedState) persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE)) .getLastUploadedManifestFile(); final RemotePublishRequest remotePublishRequest = new RemotePublishRequest( diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemoteStatePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemoteStatePublishRequest.java new file mode 100644 index 0000000000000..5667e6d67d062 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemoteStatePublishRequest.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.coordination; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.gateway.remote.ClusterMetadataManifest; + +import java.util.Objects; + +/** + * PublishRequest created by downloading the accepted {@link ClusterState} from Remote Store, using the published {@link ClusterMetadataManifest} + * + * @opensearch.internal + */ +public class RemoteStatePublishRequest extends PublishRequest { + private final ClusterMetadataManifest manifest; + + public RemoteStatePublishRequest(ClusterState acceptedState, ClusterMetadataManifest acceptedManifest) { + super(acceptedState); + this.manifest = acceptedManifest; + } + + public ClusterMetadataManifest getAcceptedManifest() { + return manifest; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + RemoteStatePublishRequest that = (RemoteStatePublishRequest) o; + return Objects.equals(manifest, that.manifest); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), manifest); + } + + @Override + public String toString() { + return "RemoteStatePublishRequest{" + super.toString() + "manifest=" + manifest + "} "; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index bd56c9e1757c6..b3836edcd7d6c 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -697,8 +697,18 @@ public String getLastUploadedManifestFile() { return lastUploadedManifestFile; } + @Override + public ClusterMetadataManifest getLastAcceptedManifest() { + return lastAcceptedManifest; + } + @Override public void setLastAcceptedState(ClusterState clusterState) { + // for non leader node, update the lastAcceptedClusterState + if (clusterState == null || clusterState.getNodes().isLocalNodeElectedClusterManager() == false) { + lastAcceptedState = clusterState; + return; + } try { final RemoteClusterStateManifestInfo manifestDetails; // Decide the codec version @@ -735,7 +745,7 @@ assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == } assert verifyManifestAndClusterState(manifestDetails.getClusterMetadataManifest(), clusterState) == true : "Manifest and ClusterState are not in sync"; - lastAcceptedManifest = manifestDetails.getClusterMetadataManifest(); + setLastAcceptedManifest(manifestDetails.getClusterMetadataManifest()); lastAcceptedState = clusterState; lastUploadedManifestFile = manifestDetails.getManifestFileName(); } catch (Exception e) { @@ -744,6 +754,11 @@ assert verifyManifestAndClusterState(manifestDetails.getClusterMetadataManifest( } } + @Override + public void setLastAcceptedManifest(ClusterMetadataManifest manifest) { + this.lastAcceptedManifest = manifest; + } + @Override public PersistedStateStats getStats() { return remoteClusterStateService.getUploadStats(); @@ -767,7 +782,7 @@ private boolean shouldWriteFullClusterState(ClusterState clusterState, int codec assert lastAcceptedManifest == null || lastAcceptedManifest.getCodecVersion() <= codecVersion; if (lastAcceptedState == null || lastAcceptedManifest == null - || lastAcceptedState.term() != clusterState.term() + || (remoteClusterStateService.isRemotePublicationEnabled() == false && lastAcceptedState.term() != clusterState.term()) || lastAcceptedManifest.getOpensearchVersion() != Version.CURRENT || lastAcceptedManifest.getCodecVersion() != codecVersion) { return true; @@ -781,19 +796,32 @@ public void markLastAcceptedStateAsCommitted() { assert lastAcceptedState != null : "Last accepted state is not present"; assert lastAcceptedManifest != null : "Last accepted manifest is not present"; ClusterState clusterState = lastAcceptedState; - if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false - && lastAcceptedState.metadata().clusterUUIDCommitted() == false) { + boolean shouldCommitVotingConfig = shouldCommitVotingConfig(); + boolean isClusterUUIDUnknown = lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID); + boolean isClusterUUIDCommitted = lastAcceptedState.metadata().clusterUUIDCommitted(); + if (shouldCommitVotingConfig || (isClusterUUIDUnknown == false && isClusterUUIDCommitted == false)) { Metadata.Builder metadataBuilder = Metadata.builder(lastAcceptedState.metadata()); - metadataBuilder.clusterUUIDCommitted(true); + if (shouldCommitVotingConfig) { + metadataBuilder = commitVotingConfiguration(lastAcceptedState); + } + if (isClusterUUIDUnknown == false && isClusterUUIDCommitted == false) { + metadataBuilder.clusterUUIDCommitted(true); + } clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build(); } - final RemoteClusterStateManifestInfo committedManifestDetails = remoteClusterStateService.markLastStateAsCommitted( - clusterState, - lastAcceptedManifest - ); - lastAcceptedManifest = committedManifestDetails.getClusterMetadataManifest(); + if (clusterState.getNodes().isLocalNodeElectedClusterManager()) { + final RemoteClusterStateManifestInfo committedManifestDetails = remoteClusterStateService.markLastStateAsCommitted( + clusterState, + lastAcceptedManifest, + shouldCommitVotingConfig + ); + assert committedManifestDetails != null; + setLastAcceptedManifest(committedManifestDetails.getClusterMetadataManifest()); + lastUploadedManifestFile = committedManifestDetails.getManifestFileName(); + } else { + setLastAcceptedManifest(ClusterMetadataManifest.builder(lastAcceptedManifest).committed(true).build()); + } lastAcceptedState = clusterState; - lastUploadedManifestFile = committedManifestDetails.getManifestFileName(); } catch (Exception e) { handleExceptionOnWrite(e); } @@ -804,6 +832,10 @@ public void close() throws IOException { remoteClusterStateService.close(); } + private boolean shouldCommitVotingConfig() { + return !lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()); + } + private void handleExceptionOnWrite(Exception e) { throw ExceptionsHelper.convertToRuntime(e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 3425550a9f548..e504c5abb46d3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -363,12 +363,20 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat * * @return {@link RemoteClusterStateManifestInfo} object containing uploaded manifest detail */ - @Nullable public RemoteClusterStateManifestInfo writeIncrementalMetadata( ClusterState previousClusterState, ClusterState clusterState, ClusterMetadataManifest previousManifest ) throws IOException { + if (previousClusterState == null) { + throw new IllegalArgumentException("previousClusterState cannot be null"); + } + if (clusterState == null) { + throw new IllegalArgumentException("clusterState cannot be null"); + } + if (previousManifest == null) { + throw new IllegalArgumentException("previousManifest cannot be null"); + } logger.trace("WRITING INCREMENTAL STATE"); final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); @@ -376,7 +384,6 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( logger.error("Local node is not elected cluster manager. Exiting"); return null; } - assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term(); boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles(); @@ -949,18 +956,41 @@ public RemoteClusterStateCleanupManager getCleanupManager() { } @Nullable - public RemoteClusterStateManifestInfo markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) - throws IOException { + public RemoteClusterStateManifestInfo markLastStateAsCommitted( + ClusterState clusterState, + ClusterMetadataManifest previousManifest, + boolean commitVotingConfig + ) throws IOException { assert clusterState != null : "Last accepted cluster state is not set"; if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); return null; } assert previousManifest != null : "Last cluster metadata manifest is not set"; + UploadedMetadataAttribute uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata(); + if (commitVotingConfig) { + // update the coordination metadata if voting config is committed + uploadedCoordinationMetadata = writeMetadataInParallel( + clusterState, + emptyList(), + emptyMap(), + emptyMap(), + true, + false, + false, + false, + false, + false, + emptyMap(), + false, + emptyList(), + null + ).uploadedCoordinationMetadata; + } UploadedMetadataResults uploadedMetadataResults = new UploadedMetadataResults( previousManifest.getIndices(), previousManifest.getCustomMetadataMap(), - previousManifest.getCoordinationMetadata(), + uploadedCoordinationMetadata, previousManifest.getSettingsMetadata(), previousManifest.getTemplatesMetadata(), previousManifest.getTransientSettingsMetadata(), @@ -1762,6 +1792,10 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } } + public boolean isRemotePublicationEnabled() { + return this.isPublicationEnabled; + } + public void setRemoteStateReadTimeout(TimeValue remoteStateReadTimeout) { this.remoteStateReadTimeout = remoteStateReadTimeout; } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index d003b54adcccc..32cb95e0c04f6 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -57,6 +57,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -72,20 +73,28 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; public class CoordinationStateTests extends OpenSearchTestCase { private DiscoveryNode node1; private DiscoveryNode node2; private DiscoveryNode node3; + private DiscoveryNode nodeWithPub; private ClusterState initialStateNode1; + private ClusterState initialStateNode2; private PersistedState ps1; private PersistedStateRegistry psr1; @@ -99,16 +108,18 @@ public void setupNodes() { node1 = createNode("node1"); node2 = createNode("node2"); node3 = createNode("node3"); + nodeWithPub = createNode( + "nodeWithPub", + Map.of( + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + "", + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, + "" + ) + ); initialStateNode1 = clusterState(0L, 0L, node1, VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 42L); - ClusterState initialStateNode2 = clusterState( - 0L, - 0L, - node2, - VotingConfiguration.EMPTY_CONFIG, - VotingConfiguration.EMPTY_CONFIG, - 42L - ); + initialStateNode2 = clusterState(0L, 0L, node2, VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 42L); ClusterState initialStateNode3 = clusterState( 0L, 0L, @@ -128,6 +139,10 @@ public void setupNodes() { } public static DiscoveryNode createNode(String id) { + return createNode(id, Collections.emptyMap()); + } + + public static DiscoveryNode createNode(String id, Map attributes) { final TransportAddress address = buildNewFakeTransportAddress(); return new DiscoveryNode( "", @@ -136,7 +151,7 @@ public static DiscoveryNode createNode(String id) { address.address().getHostString(), address.getAddress(), address, - Collections.emptyMap(), + attributes, DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT ); @@ -913,7 +928,7 @@ public void testSafety() { public void testHandlePrePublishAndCommitWhenRemoteStateDisabled() { final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1); - final PersistedStateRegistry persistedStateRegistrySpy = Mockito.spy(persistedStateRegistry); + final PersistedStateRegistry persistedStateRegistrySpy = spy(persistedStateRegistry); final CoordinationState coordinationState = createCoordinationState(persistedStateRegistrySpy, node1, Settings.EMPTY); final VotingConfiguration initialConfig = VotingConfiguration.of(node1); final ClusterState clusterState = clusterState(0L, 0L, node1, initialConfig, initialConfig, 42L); @@ -932,7 +947,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateDisabled() { public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final VotingConfiguration initialConfig = VotingConfiguration.of(node1); - final ClusterState clusterState = clusterState(0L, 0L, node1, initialConfig, initialConfig, 42L); + final ClusterState clusterState = clusterStateWithClusterManager(0L, 0L, node1, node1, initialConfig, initialConfig, 42L); final String previousClusterUUID = "prev-cluster-uuid"; final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() .clusterTerm(0L) @@ -948,8 +963,9 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep .previousClusterUUID(randomAlphaOfLength(10)) .clusterUUIDCommitted(true) .build(); - Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION)) - .thenReturn(new RemoteClusterStateManifestInfo(manifest, "path/to/manifest")); + when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION)).thenReturn( + new RemoteClusterStateManifestInfo(manifest, "path/to/manifest") + ); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1); @@ -958,40 +974,21 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep new RemotePersistedState(remoteClusterStateService, previousClusterUUID) ); - String randomRepoName = "randomRepoName"; - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - randomRepoName - ); - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - randomRepoName - ); - - Settings settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, randomRepoName) - .put(stateRepoTypeAttributeKey, FsRepository.TYPE) - .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .build(); - - final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, settings); + final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, remoteStateSettings()); coordinationState.handlePrePublish(clusterState); Mockito.verify(remoteClusterStateService, Mockito.times(1)) .writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); assertThat(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState(), equalTo(clusterState)); - Mockito.when(remoteClusterStateService.markLastStateAsCommitted(any(), any())) - .thenReturn(new RemoteClusterStateManifestInfo(manifest, "path/to/manifest")); + when(remoteClusterStateService.markLastStateAsCommitted(any(), any(), eq(false))).thenReturn( + new RemoteClusterStateManifestInfo(manifest, "path/to/manifest") + ); coordinationState.handlePreCommit(); ClusterState committedClusterState = ClusterState.builder(clusterState) .metadata(Metadata.builder(clusterState.metadata()).clusterUUIDCommitted(true).build()) .build(); - // Mockito.verify(remoteClusterStateService, Mockito.times(1)).markLastStateAsCommitted(committedClusterState, manifest); ArgumentCaptor clusterStateCaptor = ArgumentCaptor.forClass(ClusterState.class); - verify(remoteClusterStateService, times(1)).markLastStateAsCommitted(clusterStateCaptor.capture(), any()); + verify(remoteClusterStateService, times(1)).markLastStateAsCommitted(clusterStateCaptor.capture(), any(), eq(false)); assertThat(clusterStateCaptor.getValue().metadata().indices(), equalTo(committedClusterState.metadata().indices())); assertThat(clusterStateCaptor.getValue().metadata().clusterUUID(), equalTo(committedClusterState.metadata().clusterUUID())); assertThat(clusterStateCaptor.getValue().stateUUID(), equalTo(committedClusterState.stateUUID())); @@ -1006,6 +1003,271 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep ); } + public void testHandlePublishRequestOnFollowerWhenRemotePublicationEnabled() { + final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); + // cluster manager is node1 and node2 is a follower node + VotingConfiguration initialConfig = VotingConfiguration.of(node1); + ClusterState state1 = clusterState( + 0L, + 0L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + initialConfig, + 42L + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(0L, initialStateNode2)); + persistedStateRegistry.addPersistedState( + PersistedStateType.REMOTE, + new RemotePersistedState(remoteClusterStateService, state1.metadata().clusterUUID()) + ); + + final CoordinationState coordinationState = createCoordinationState( + persistedStateRegistry, + nodeWithPub, + remotePublicationSettings() + ); + coordinationState.setInitialState(state1); + long newTerm = randomLongBetween(1, 10); + StartJoinRequest startJoinRequest = new StartJoinRequest(nodeWithPub, newTerm); + + coordinationState.handleStartJoin(startJoinRequest); + + ClusterState state2 = setValue( + ClusterState.builder(state1) + .metadata( + Metadata.builder(state1.metadata()) + .coordinationMetadata(CoordinationMetadata.builder(state1.coordinationMetadata()).term(newTerm).build()) + .build() + ) + .version(randomLongBetween(1, 10)) + .build(), + 43L + ); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(state2.version()) + .clusterUUID(state2.metadata().clusterUUID()) + .nodeId(node1.getId()) + .stateUUID(randomAlphaOfLength(10)) + .opensearchVersion(Version.CURRENT) + .committed(false) + .codecVersion(1) + .globalMetadataFileName(randomAlphaOfLength(10)) + .indices(Collections.emptyList()) + .previousClusterUUID(randomAlphaOfLength(10)) + .clusterUUIDCommitted(true) + .build(); + + PublishResponse publishResponse = coordinationState.handlePublishRequest(new RemoteStatePublishRequest(state2, manifest)); + assertEquals(state2.term(), publishResponse.getTerm()); + assertEquals(state2.version(), publishResponse.getVersion()); + verifyNoInteractions(remoteClusterStateService); + assertEquals(state2, persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState()); + assertEquals(manifest, persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedManifest()); + } + + public void testHandleCommitOnFollowerNodeWhenRemotePublicationEnabled() { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + VotingConfiguration initialConfig = VotingConfiguration.of(node1, nodeWithPub); + ClusterState state1 = clusterState( + 0L, + 0L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + initialConfig, + 42L + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(0L, initialStateNode2)); + persistedStateRegistry.addPersistedState( + PersistedStateType.REMOTE, + new RemotePersistedState(remoteClusterStateService, state1.metadata().clusterUUID()) + ); + + final CoordinationState coordinationState = createCoordinationState( + persistedStateRegistry, + nodeWithPub, + remotePublicationSettings() + ); + coordinationState.setInitialState(state1); + long newTerm = randomLongBetween(1, 10); + StartJoinRequest startJoinRequest = new StartJoinRequest(nodeWithPub, newTerm); + + Join v1 = cs1.handleStartJoin(startJoinRequest); + Join v2 = coordinationState.handleStartJoin(startJoinRequest); + assertTrue(coordinationState.handleJoin(v1)); + assertTrue(coordinationState.handleJoin(v2)); + assertTrue(coordinationState.electionWon()); + VotingConfiguration newConfig = VotingConfiguration.of(node1, nodeWithPub, node3); + ClusterState state2 = clusterState( + startJoinRequest.getTerm(), + 2L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + newConfig, + 7L + ); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(state2.version()) + .clusterUUID(state2.metadata().clusterUUID()) + .nodeId(node1.getId()) + .stateUUID(randomAlphaOfLength(10)) + .opensearchVersion(Version.CURRENT) + .committed(false) + .codecVersion(1) + .globalMetadataFileName(randomAlphaOfLength(10)) + .indices(Collections.emptyList()) + .previousClusterUUID(randomAlphaOfLength(10)) + .clusterUUIDCommitted(true) + .build(); + + PublishRequest publishRequest = coordinationState.handleClientValue(state2); + coordinationState.handlePublishRequest(new RemoteStatePublishRequest(publishRequest.getAcceptedState(), manifest)); + ApplyCommitRequest applyCommitRequest = new ApplyCommitRequest(node2, state2.term(), state2.version()); + coordinationState.handleCommit(applyCommitRequest); + verifyNoInteractions(remoteClusterStateService); + assertTrue( + persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState().metadata().clusterUUIDCommitted() + ); + assertTrue(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedManifest().isCommitted()); + assertEquals(coordinationState.getLastCommittedConfiguration(), newConfig); + } + + public void testRemotePersistedStateResetsForPublicationEnabledAfterLocalPublication() { + final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); + // cluster manager is node1 and nodeWithPub is a follower node + VotingConfiguration initialConfig = VotingConfiguration.of(node1); + ClusterState state1 = clusterState( + 0L, + 0L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + initialConfig, + 42L + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(0L, initialStateNode2)); + persistedStateRegistry.addPersistedState( + PersistedStateType.REMOTE, + new RemotePersistedState(remoteClusterStateService, state1.metadata().clusterUUID()) + ); + + final CoordinationState coordinationState = createCoordinationState( + persistedStateRegistry, + nodeWithPub, + remotePublicationSettings() + ); + coordinationState.setInitialState(state1); + long newTerm = randomLongBetween(1, 10); + StartJoinRequest startJoinRequest = new StartJoinRequest(nodeWithPub, newTerm); + + coordinationState.handleStartJoin(startJoinRequest); + + ClusterState state2 = setValue( + ClusterState.builder(state1) + .metadata( + Metadata.builder(state1.metadata()) + .coordinationMetadata(CoordinationMetadata.builder(state1.coordinationMetadata()).term(newTerm).build()) + .build() + ) + .version(randomLongBetween(1, 10)) + .build(), + 43L + ); + + PublishResponse publishResponse = coordinationState.handlePublishRequest(new PublishRequest(state2)); + assertEquals(state2.term(), publishResponse.getTerm()); + assertEquals(state2.version(), publishResponse.getVersion()); + verifyNoInteractions(remoteClusterStateService); + assertNull(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState()); + assertNull(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedManifest()); + } + + public void testHandleCommitOnFollowerNodeWhenRemotePublicationEnabledWithNullRemotePersistedState() { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + VotingConfiguration initialConfig = VotingConfiguration.of(node1, nodeWithPub); + ClusterState state1 = clusterState( + 0L, + 0L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + initialConfig, + 42L + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(0L, initialStateNode2)); + persistedStateRegistry.addPersistedState( + PersistedStateType.REMOTE, + new RemotePersistedState(remoteClusterStateService, state1.metadata().clusterUUID()) + ); + + final CoordinationState coordinationState = createCoordinationState( + persistedStateRegistry, + nodeWithPub, + remotePublicationSettings() + ); + coordinationState.setInitialState(state1); + long newTerm = randomLongBetween(1, 10); + StartJoinRequest startJoinRequest = new StartJoinRequest(nodeWithPub, newTerm); + + Join v1 = cs1.handleStartJoin(startJoinRequest); + Join v2 = coordinationState.handleStartJoin(startJoinRequest); + assertTrue(coordinationState.handleJoin(v1)); + assertTrue(coordinationState.handleJoin(v2)); + assertTrue(coordinationState.electionWon()); + VotingConfiguration newConfig = VotingConfiguration.of(node1, nodeWithPub, node3); + ClusterState state2 = clusterState( + startJoinRequest.getTerm(), + 2L, + DiscoveryNodes.builder() + .add(node1) + .add(nodeWithPub) + .clusterManagerNodeId(node1.getId()) + .localNodeId(nodeWithPub.getId()) + .build(), + initialConfig, + newConfig, + 7L + ); + + PublishRequest publishRequest = coordinationState.handleClientValue(state2); + coordinationState.handlePublishRequest(new PublishRequest(publishRequest.getAcceptedState())); + assertNull(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState()); + assertNull(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedManifest()); + ApplyCommitRequest applyCommitRequest = new ApplyCommitRequest(node2, state2.term(), state2.version()); + PersistedState spyRPS = spy(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE)); + coordinationState.handleCommit(applyCommitRequest); + verifyNoInteractions(spyRPS); + verifyNoInteractions(remoteClusterStateService); + } + public void testIsRemotePublicationEnabled_WithInconsistentSettings() { // create settings with remote state disabled but publication enabled Settings settings = Settings.builder() @@ -1042,6 +1304,30 @@ public static ClusterState clusterState( ); } + public static ClusterState clusterStateWithClusterManager( + long term, + long version, + DiscoveryNode localNode, + DiscoveryNode clusterManagerNode, + VotingConfiguration lastCommittedConfig, + VotingConfiguration lastAcceptedConfig, + long value + ) { + return clusterState( + term, + version, + DiscoveryNodes.builder() + .add(localNode) + .add(clusterManagerNode) + .localNodeId(localNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + .build(), + lastCommittedConfig, + lastAcceptedConfig, + value + ); + } + public static ClusterState clusterState( long term, long version, @@ -1090,4 +1376,30 @@ private static PersistedStateRegistry createPersistedStateRegistry(ClusterState persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, new InMemoryPersistedState(0L, clusterState)); return persistedStateRegistry; } + + private static Settings remoteStateSettings() { + String randomRepoName = "randomRepoName"; + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + randomRepoName + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + randomRepoName + ); + + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, randomRepoName) + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + return settings; + } + + private static Settings remotePublicationSettings() { + return Settings.builder().put(remoteStateSettings()).put(REMOTE_PUBLICATION_SETTING_KEY, true).build(); + } } diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 9972bbfff5d66..5ea5241762753 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -115,6 +115,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -210,11 +211,15 @@ public void testSetCurrentTerm() throws IOException { } private ClusterState createClusterState(long version, Metadata metadata) { - return ClusterState.builder(clusterName) - .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).build()) - .version(version) - .metadata(metadata) - .build(); + return createClusterState(version, metadata, false); + } + + private ClusterState createClusterState(long version, Metadata metadata, boolean isClusterManagerNode) { + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()); + if (isClusterManagerNode) { + nodesBuilder.clusterManagerNodeId(localNode.getId()); + } + return ClusterState.builder(clusterName).nodes(nodesBuilder.build()).version(version).metadata(metadata).build(); } private ClusterState createClusterStateWithNodes(long version, Metadata metadata) { @@ -225,7 +230,12 @@ private ClusterState createClusterStateWithNodes(long version, Metadata metadata Sets.newHashSet(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE), Version.V_2_13_0 ); - DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).add(oldNode).build(); + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder() + .add(localNode) + .localNodeId(localNode.getId()) + .clusterManagerNodeId(localNode.getId()) + .add(oldNode) + .build(); return ClusterState.builder(clusterName).nodes(discoveryNodes).version(version).metadata(metadata).build(); } @@ -762,7 +772,8 @@ public void testRemotePersistedState() throws IOException { final long clusterTerm = randomNonNegativeLong(); final ClusterState clusterState = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build(), + true ); remotePersistedState.setLastAcceptedState(clusterState); @@ -773,7 +784,8 @@ public void testRemotePersistedState() throws IOException { final ClusterState secondClusterState = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build(), + true ); remotePersistedState.setLastAcceptedState(secondClusterState); @@ -783,11 +795,11 @@ public void testRemotePersistedState() throws IOException { assertThat(remotePersistedState.getLastAcceptedState(), equalTo(secondClusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); - when(remoteClusterStateService.markLastStateAsCommitted(Mockito.any(), Mockito.any())).thenReturn( + when(remoteClusterStateService.markLastStateAsCommitted(Mockito.any(), Mockito.any(), eq(false))).thenReturn( new RemoteClusterStateManifestInfo(manifest, "path/to/manifest") ); remotePersistedState.markLastAcceptedStateAsCommitted(); - Mockito.verify(remoteClusterStateService, times(1)).markLastStateAsCommitted(Mockito.any(), Mockito.any()); + Mockito.verify(remoteClusterStateService, times(1)).markLastStateAsCommitted(Mockito.any(), Mockito.any(), eq(false)); assertThat(remotePersistedState.getLastAcceptedState(), equalTo(secondClusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); @@ -825,7 +837,8 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx ClusterState clusterState2 = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(1L).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(1L).build()).build(), + true ); final ClusterMetadataManifest manifest2 = ClusterMetadataManifest.builder() .clusterTerm(1L) @@ -840,7 +853,8 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx ClusterState clusterState3 = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(1L).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(1L).build()).build(), + true ); Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())) .thenReturn(new RemoteClusterStateManifestInfo(manifest2, "path/to/manifest3")); @@ -849,6 +863,73 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx } + public void testRemotePersistentState_FollowerNode() throws IOException { + final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(5L) + .committed(false) + .build(); + final String previousClusterUUID = "prev-cluster-uuid"; + RemotePersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); + + assertNull(remotePersistedState.getLastAcceptedState()); + assertNull(remotePersistedState.getLastAcceptedManifest()); + assertEquals(0, remotePersistedState.getCurrentTerm()); + + final long clusterTerm = randomNonNegativeLong(); + final ClusterState clusterState = createClusterState( + randomNonNegativeLong(), + Metadata.builder() + .coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()) + .clusterUUIDCommitted(true) + .build(), + false + ); + + remotePersistedState.setLastAcceptedState(clusterState); + remotePersistedState.setLastAcceptedManifest(manifest); + Mockito.verify(remoteClusterStateService, never()) + .writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + + assertEquals(clusterState, remotePersistedState.getLastAcceptedState()); + assertEquals(clusterTerm, remotePersistedState.getCurrentTerm()); + assertEquals(manifest, remotePersistedState.getLastAcceptedManifest()); + + final ClusterState secondClusterState = createClusterState( + randomNonNegativeLong(), + Metadata.builder() + .coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()) + .clusterUUIDCommitted(false) + .build(), + false + ); + + remotePersistedState.setLastAcceptedState(secondClusterState); + Mockito.verify(remoteClusterStateService, never()) + .writeFullMetadata(secondClusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + + assertEquals(secondClusterState, remotePersistedState.getLastAcceptedState()); + assertEquals(clusterTerm, remotePersistedState.getCurrentTerm()); + assertFalse(remotePersistedState.getLastAcceptedManifest().isCommitted()); + + remotePersistedState.markLastAcceptedStateAsCommitted(); + Mockito.verify(remoteClusterStateService, never()).markLastStateAsCommitted(Mockito.any(), Mockito.any(), eq(false)); + + assertEquals(secondClusterState, remotePersistedState.getLastAcceptedState()); + assertEquals(clusterTerm, remotePersistedState.getCurrentTerm()); + assertFalse(remotePersistedState.getLastAcceptedState().metadata().clusterUUIDCommitted()); + assertTrue(remotePersistedState.getLastAcceptedManifest().isCommitted()); + + final ClusterState thirdClusterState = ClusterState.builder(secondClusterState) + .metadata(Metadata.builder(secondClusterState.getMetadata()).clusterUUID(randomAlphaOfLength(10)).build()) + .build(); + remotePersistedState.setLastAcceptedState(thirdClusterState); + remotePersistedState.markLastAcceptedStateAsCommitted(); + assertTrue(remotePersistedState.getLastAcceptedState().metadata().clusterUUIDCommitted()); + assertTrue(remotePersistedState.getLastAcceptedManifest().isCommitted()); + } + public void testRemotePersistedStateNotCommitted() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final String previousClusterUUID = "prev-cluster-uuid"; @@ -875,7 +956,8 @@ public void testRemotePersistedStateNotCommitted() throws IOException { final long clusterTerm = randomNonNegativeLong(); ClusterState clusterState = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build(), + true ); clusterState = ClusterState.builder(clusterState) .metadata(Metadata.builder(clusterState.getMetadata()).clusterUUID(randomAlphaOfLength(10)).clusterUUIDCommitted(false).build()) @@ -901,7 +983,8 @@ public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOExcept final long clusterTerm = randomNonNegativeLong(); final ClusterState clusterState = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build(), + true ); assertThrows(OpenSearchException.class, () -> remotePersistedState.setLastAcceptedState(clusterState)); @@ -924,7 +1007,8 @@ public void testRemotePersistedStateFailureStats() throws IOException { final long clusterTerm = randomNonNegativeLong(); final ClusterState clusterState = createClusterState( randomNonNegativeLong(), - Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build() + Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(clusterTerm).build()).build(), + true ); assertThrows(OpenSearchException.class, () -> remotePersistedState.setLastAcceptedState(clusterState)); @@ -1025,7 +1109,8 @@ public void testGatewayForRemoteStateForNodeReplacement() throws IOException { false ) .clusterUUID(randomAlphaOfLength(10)) - .build() + .build(), + false ); when(remoteClusterStateService.getLastKnownUUIDFromRemote(clusterName.value())).thenReturn( previousState.metadata().clusterUUID() @@ -1071,7 +1156,8 @@ public void testGatewayForRemoteStateForNodeReboot() throws IOException { .coordinationMetadata(CoordinationMetadata.builder().term(randomLong()).build()) .put(indexMetadata, false) .clusterUUID(randomAlphaOfLength(10)) - .build() + .build(), + false ); gateway = newGatewayForRemoteState( remoteClusterStateService, @@ -1117,7 +1203,8 @@ public void testGatewayForRemoteStateForInitialBootstrapBlocksApplied() throws I .put(indexMetadata, false) .clusterUUID(ClusterState.UNKNOWN_UUID) .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), true).build()) - .build() + .build(), + false ) ).nodes(DiscoveryNodes.EMPTY_NODES).build(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 608cc2e12b055..e875b1c5dc64e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -608,20 +608,20 @@ public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOExc final RemoteClusterStateManifestInfo manifestDetails = remoteClusterStateService.writeIncrementalMetadata( clusterState, clusterState, - null + ClusterMetadataManifest.builder().build() ); Assert.assertThat(manifestDetails, nullValue()); assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } - public void testFailWriteIncrementalMetadataWhenTermChanged() { + public void testFailWriteIncrementalMetadataWhenManifestNull() { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(2L).build(); final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) .build(); assertThrows( - AssertionError.class, + IllegalArgumentException.class, () -> remoteClusterStateService.writeIncrementalMetadata(previousClusterState, clusterState, null) ); } @@ -2520,7 +2520,7 @@ public void testMarkLastStateAsCommittedSuccess() throws IOException { List indices = List.of(uploadedIndexMetadata); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(indices).build(); - final ClusterMetadataManifest manifest = remoteClusterStateService.markLastStateAsCommitted(clusterState, previousManifest) + final ClusterMetadataManifest manifest = remoteClusterStateService.markLastStateAsCommitted(clusterState, previousManifest, false) .getClusterMetadataManifest(); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() From 89915b65dc51126189dd42d74c82d2cbac0cb154 Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:37:20 +0530 Subject: [PATCH 078/357] Optimise memory for rest cluster health calls with in-line shard aggregations for levels cluster and indices. (#15492) Signed-off-by: Swetha Guptha --- CHANGELOG.md | 1 + .../opensearch/cluster/ClusterHealthIT.java | 163 ++++++++++++++++++ .../RemoteStoreMigrationSettingsUpdateIT.java | 3 +- .../cluster/health/ClusterHealthRequest.java | 25 +++ .../health/ClusterHealthRequestBuilder.java | 5 + .../cluster/health/ClusterHealthResponse.java | 43 +++++ .../health/TransportClusterHealthAction.java | 14 +- .../stats/TransportClusterStatsAction.java | 3 +- .../cluster/health/ClusterIndexHealth.java | 120 ++++++++++++- .../cluster/health/ClusterShardHealth.java | 48 ++++-- .../cluster/health/ClusterStateHealth.java | 104 ++++++++++- .../routing/allocation/AllocationService.java | 7 +- .../cluster/RestClusterHealthAction.java | 1 + .../health/ClusterIndexHealthTests.java | 27 ++- .../health/ClusterShardHealthTests.java | 122 +++++++++++++ .../health/ClusterStateHealthTests.java | 29 +++- 16 files changed, 674 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deb9b7d0bcad7..e09dc9d192b4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) +- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java index 0304e00a49070..33f101cce2382 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java @@ -37,6 +37,8 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.health.ClusterIndexHealth; +import org.opensearch.cluster.health.ClusterShardHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.service.ClusterService; @@ -49,6 +51,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -439,4 +442,164 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS completionFuture.actionGet(TimeValue.timeValueSeconds(30)); } } + + public void testHealthWithClusterLevelAppliedAtTransportLayer() { + createIndex( + "test1", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() + ); + ensureGreen(); + ClusterHealthResponse healthResponse = client().admin() + .cluster() + .prepareHealth() + .setApplyLevelAtTransportLayer(true) + .execute() + .actionGet(); + assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus()); + assertTrue(healthResponse.getIndices().isEmpty()); + assertEquals(1, healthResponse.getActiveShards()); + assertEquals(1, healthResponse.getActivePrimaryShards()); + assertEquals(0, healthResponse.getUnassignedShards()); + assertEquals(0, healthResponse.getInitializingShards()); + assertEquals(0, healthResponse.getRelocatingShards()); + assertEquals(0, healthResponse.getDelayedUnassignedShards()); + } + + public void testHealthWithIndicesLevelAppliedAtTransportLayer() { + createIndex( + "test1", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() + ); + ensureGreen(); + ClusterHealthResponse healthResponse = client().admin() + .cluster() + .prepareHealth() + .setLevel("indices") + .setApplyLevelAtTransportLayer(true) + .execute() + .actionGet(); + assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus()); + + assertEquals(1, healthResponse.getActiveShards()); + assertEquals(1, healthResponse.getActivePrimaryShards()); + assertEquals(0, healthResponse.getUnassignedShards()); + assertEquals(0, healthResponse.getInitializingShards()); + assertEquals(0, healthResponse.getRelocatingShards()); + assertEquals(0, healthResponse.getDelayedUnassignedShards()); + + Map indices = healthResponse.getIndices(); + assertFalse(indices.isEmpty()); + assertEquals(1, indices.size()); + for (Map.Entry indicesHealth : indices.entrySet()) { + String indexName = indicesHealth.getKey(); + assertEquals("test1", indexName); + ClusterIndexHealth indicesHealthValue = indicesHealth.getValue(); + assertEquals(1, indicesHealthValue.getActiveShards()); + assertEquals(1, indicesHealthValue.getActivePrimaryShards()); + assertEquals(0, indicesHealthValue.getInitializingShards()); + assertEquals(0, indicesHealthValue.getUnassignedShards()); + assertEquals(0, indicesHealthValue.getDelayedUnassignedShards()); + assertEquals(0, indicesHealthValue.getRelocatingShards()); + assertEquals(ClusterHealthStatus.GREEN, indicesHealthValue.getStatus()); + assertTrue(indicesHealthValue.getShards().isEmpty()); + } + } + + public void testHealthWithShardLevelAppliedAtTransportLayer() { + int dataNodes = internalCluster().getDataNodeNames().size(); + int greenClusterReplicaCount = dataNodes - 1; + int yellowClusterReplicaCount = dataNodes; + + createIndex( + "test1", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount) + .build() + ); + ensureGreen(TimeValue.timeValueSeconds(120), "test1"); + createIndex( + "test2", + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, greenClusterReplicaCount) + .build() + ); + ensureGreen(TimeValue.timeValueSeconds(120)); + client().admin() + .indices() + .prepareUpdateSettings() + .setIndices("test2") + .setSettings(Settings.builder().put("index.number_of_replicas", yellowClusterReplicaCount).build()) + .execute() + .actionGet(); + ClusterHealthResponse healthResponse = client().admin() + .cluster() + .prepareHealth() + .setLevel("shards") + .setApplyLevelAtTransportLayer(true) + .execute() + .actionGet(); + assertEquals(ClusterHealthStatus.YELLOW, healthResponse.getStatus()); + + assertEquals(2 * dataNodes, healthResponse.getActiveShards()); + assertEquals(2, healthResponse.getActivePrimaryShards()); + assertEquals(1, healthResponse.getUnassignedShards()); + assertEquals(0, healthResponse.getInitializingShards()); + assertEquals(0, healthResponse.getRelocatingShards()); + assertEquals(0, healthResponse.getDelayedUnassignedShards()); + + Map indices = healthResponse.getIndices(); + assertFalse(indices.isEmpty()); + assertEquals(2, indices.size()); + for (Map.Entry indicesHealth : indices.entrySet()) { + String indexName = indicesHealth.getKey(); + boolean indexHasMoreReplicas = indexName.equals("test2"); + ClusterIndexHealth indicesHealthValue = indicesHealth.getValue(); + assertEquals(dataNodes, indicesHealthValue.getActiveShards()); + assertEquals(1, indicesHealthValue.getActivePrimaryShards()); + assertEquals(0, indicesHealthValue.getInitializingShards()); + assertEquals(indexHasMoreReplicas ? 1 : 0, indicesHealthValue.getUnassignedShards()); + assertEquals(0, indicesHealthValue.getDelayedUnassignedShards()); + assertEquals(0, indicesHealthValue.getRelocatingShards()); + assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, indicesHealthValue.getStatus()); + Map shards = indicesHealthValue.getShards(); + assertFalse(shards.isEmpty()); + assertEquals(1, shards.size()); + for (Map.Entry shardHealth : shards.entrySet()) { + ClusterShardHealth clusterShardHealth = shardHealth.getValue(); + assertEquals(dataNodes, clusterShardHealth.getActiveShards()); + assertEquals(indexHasMoreReplicas ? 1 : 0, clusterShardHealth.getUnassignedShards()); + assertEquals(0, clusterShardHealth.getDelayedUnassignedShards()); + assertEquals(0, clusterShardHealth.getRelocatingShards()); + assertEquals(0, clusterShardHealth.getInitializingShards()); + assertTrue(clusterShardHealth.isPrimaryActive()); + assertEquals(indexHasMoreReplicas ? ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN, clusterShardHealth.getStatus()); + } + } + } + + public void testHealthWithAwarenessAttributesLevelAppliedAtTransportLayer() { + createIndex( + "test1", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() + ); + ensureGreen(); + ClusterHealthResponse healthResponse = client().admin() + .cluster() + .prepareHealth() + .setLevel("awareness_attributes") + .setApplyLevelAtTransportLayer(true) + .execute() + .actionGet(); + assertEquals(ClusterHealthStatus.GREEN, healthResponse.getStatus()); + assertTrue(healthResponse.getIndices().isEmpty()); + assertNotNull(healthResponse.getClusterAwarenessHealth()); + assertEquals(1, healthResponse.getActiveShards()); + assertEquals(1, healthResponse.getActivePrimaryShards()); + assertEquals(0, healthResponse.getUnassignedShards()); + assertEquals(0, healthResponse.getInitializingShards()); + assertEquals(0, healthResponse.getRelocatingShards()); + assertEquals(0, healthResponse.getDelayedUnassignedShards()); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java index c701a8d92c336..4a6cb28b6db00 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java @@ -11,6 +11,7 @@ import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; +import org.opensearch.common.unit.TimeValue; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -109,7 +110,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix setDirection(REMOTE_STORE.direction); String restoredIndexName2 = TEST_INDEX + "-restored2"; restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2); - ensureGreen(restoredIndexName2); + ensureGreen(TimeValue.timeValueSeconds(90), restoredIndexName2); logger.info("Verify that restored index is non remote-backed"); assertRemoteStoreBackedIndex(restoredIndexName2); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java index ec8b01d853da6..55d3e2089b2a8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -76,6 +76,17 @@ public class ClusterHealthRequest extends ClusterManagerNodeReadRequest 0) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java index a9a3756755265..87dd77aca20c6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequestBuilder.java @@ -171,4 +171,9 @@ public final ClusterHealthRequestBuilder setEnsureNodeWeighedIn(boolean ensureNo request.ensureNodeWeighedIn(ensureNodeCommissioned); return this; } + + public ClusterHealthRequestBuilder setApplyLevelAtTransportLayer(boolean applyLevelAtTransportLayer) { + request.setApplyLevelAtTransportLayer(applyLevelAtTransportLayer); + return this; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java index 1a27f161343e8..2dcfb58c3d7b8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -237,6 +237,26 @@ public ClusterHealthResponse( this.clusterHealthStatus = clusterStateHealth.getStatus(); } + public ClusterHealthResponse( + String clusterName, + String[] concreteIndices, + ClusterHealthRequest clusterHealthRequest, + ClusterState clusterState, + int numberOfPendingTasks, + int numberOfInFlightFetch, + TimeValue taskMaxWaitingTime + ) { + this.clusterName = clusterName; + this.numberOfPendingTasks = numberOfPendingTasks; + this.numberOfInFlightFetch = numberOfInFlightFetch; + this.taskMaxWaitingTime = taskMaxWaitingTime; + this.clusterStateHealth = clusterHealthRequest.isApplyLevelAtTransportLayer() + ? new ClusterStateHealth(clusterState, concreteIndices, clusterHealthRequest.level()) + : new ClusterStateHealth(clusterState, concreteIndices); + this.clusterHealthStatus = clusterStateHealth.getStatus(); + this.delayedUnassignedShards = clusterStateHealth.getDelayedUnassignedShards(); + } + // Awareness Attribute health public ClusterHealthResponse( String clusterName, @@ -261,6 +281,29 @@ public ClusterHealthResponse( this.clusterAwarenessHealth = new ClusterAwarenessHealth(clusterState, clusterSettings, awarenessAttributeName); } + public ClusterHealthResponse( + String clusterName, + ClusterHealthRequest clusterHealthRequest, + ClusterState clusterState, + ClusterSettings clusterSettings, + String[] concreteIndices, + String awarenessAttributeName, + int numberOfPendingTasks, + int numberOfInFlightFetch, + TimeValue taskMaxWaitingTime + ) { + this( + clusterName, + concreteIndices, + clusterHealthRequest, + clusterState, + numberOfPendingTasks, + numberOfInFlightFetch, + taskMaxWaitingTime + ); + this.clusterAwarenessHealth = new ClusterAwarenessHealth(clusterState, clusterSettings, awarenessAttributeName); + } + /** * For XContent Parser and serialization tests */ diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java index f69f462372888..8f0202e6d1ed0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -52,7 +52,6 @@ import org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.NodeWeighedAwayException; -import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.WeightedRoutingUtils; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; @@ -487,7 +486,12 @@ private ClusterHealthResponse clusterHealth( TimeValue pendingTaskTimeInQueue ) { if (logger.isTraceEnabled()) { - logger.trace("Calculating health based on state version [{}]", clusterState.version()); + logger.trace( + "Calculating health based on state version [{}] for health level [{}] and applyLevelAtTransportLayer set to [{}]", + clusterState.version(), + request.level(), + request.isApplyLevelAtTransportLayer() + ); } String[] concreteIndices; @@ -496,13 +500,13 @@ private ClusterHealthResponse clusterHealth( concreteIndices = clusterState.getMetadata().getConcreteAllIndices(); return new ClusterHealthResponse( clusterState.getClusterName().value(), + request, clusterState, clusterService.getClusterSettings(), concreteIndices, awarenessAttribute, numberOfPendingTasks, numberOfInFlightFetch, - UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue ); } @@ -514,10 +518,10 @@ private ClusterHealthResponse clusterHealth( ClusterHealthResponse response = new ClusterHealthResponse( clusterState.getClusterName().value(), Strings.EMPTY_ARRAY, + request, clusterState, numberOfPendingTasks, numberOfInFlightFetch, - UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue ); response.setStatus(ClusterHealthStatus.RED); @@ -527,10 +531,10 @@ private ClusterHealthResponse clusterHealth( return new ClusterHealthResponse( clusterState.getClusterName().value(), concreteIndices, + request, clusterState, numberOfPendingTasks, numberOfInFlightFetch, - UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index be7d41a7ba75e..a49ca2035783c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -34,6 +34,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.action.FailedNodeException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.indices.stats.CommonStats; @@ -209,7 +210,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq ClusterHealthStatus clusterStatus = null; if (clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus(); + clusterStatus = new ClusterStateHealth(clusterService.state(), ClusterHealthRequest.Level.CLUSTER).getStatus(); } return new ClusterStatsNodeResponse( diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java index 19c64965e6941..77d96cb0af792 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterIndexHealth.java @@ -32,9 +32,11 @@ package org.opensearch.cluster.health; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -141,6 +143,7 @@ public final class ClusterIndexHealth implements Iterable, W private final int relocatingShards; private final int initializingShards; private final int unassignedShards; + private int delayedUnassignedShards; private final int activePrimaryShards; private final ClusterHealthStatus status; private final Map shards; @@ -163,6 +166,7 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT int computeRelocatingShards = 0; int computeInitializingShards = 0; int computeUnassignedShards = 0; + int computeDelayedUnassignedShards = 0; for (ClusterShardHealth shardHealth : shards.values()) { if (shardHealth.isPrimaryActive()) { computeActivePrimaryShards++; @@ -171,13 +175,9 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT computeRelocatingShards += shardHealth.getRelocatingShards(); computeInitializingShards += shardHealth.getInitializingShards(); computeUnassignedShards += shardHealth.getUnassignedShards(); + computeDelayedUnassignedShards += shardHealth.getDelayedUnassignedShards(); - if (shardHealth.getStatus() == ClusterHealthStatus.RED) { - computeStatus = ClusterHealthStatus.RED; - } else if (shardHealth.getStatus() == ClusterHealthStatus.YELLOW && computeStatus != ClusterHealthStatus.RED) { - // do not override an existing red - computeStatus = ClusterHealthStatus.YELLOW; - } + computeStatus = getIndexHealthStatus(shardHealth.getStatus(), computeStatus); } if (shards.isEmpty()) { // might be since none has been created yet (two phase index creation) computeStatus = ClusterHealthStatus.RED; @@ -189,6 +189,110 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.delayedUnassignedShards = computeDelayedUnassignedShards; + } + + public ClusterIndexHealth( + final IndexMetadata indexMetadata, + final IndexRoutingTable indexRoutingTable, + final ClusterHealthRequest.Level healthLevel + ) { + this.index = indexMetadata.getIndex().getName(); + this.numberOfShards = indexMetadata.getNumberOfShards(); + this.numberOfReplicas = indexMetadata.getNumberOfReplicas(); + + shards = new HashMap<>(); + + // update the index status + ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN; + int computeActivePrimaryShards = 0; + int computeActiveShards = 0; + int computeRelocatingShards = 0; + int computeInitializingShards = 0; + int computeUnassignedShards = 0; + int computeDelayedUnassignedShards = 0; + + boolean isShardLevelHealthRequired = healthLevel == ClusterHealthRequest.Level.SHARDS; + if (isShardLevelHealthRequired) { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + int shardId = indexShardRoutingTable.shardId().id(); + ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, indexShardRoutingTable); + if (shardHealth.isPrimaryActive()) { + computeActivePrimaryShards++; + } + computeActiveShards += shardHealth.getActiveShards(); + computeRelocatingShards += shardHealth.getRelocatingShards(); + computeInitializingShards += shardHealth.getInitializingShards(); + computeUnassignedShards += shardHealth.getUnassignedShards(); + computeDelayedUnassignedShards += shardHealth.getDelayedUnassignedShards(); + computeStatus = getIndexHealthStatus(shardHealth.getStatus(), computeStatus); + shards.put(shardId, shardHealth); + } + } else { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + int activeShardsPerShardId = 0; + + List shardRoutings = indexShardRoutingTable.shards(); + int shardRoutingCountPerShardId = shardRoutings.size(); + for (int index = 0; index < shardRoutingCountPerShardId; index++) { + ShardRouting shardRouting = shardRoutings.get(index); + if (shardRouting.active()) { + computeActiveShards++; + activeShardsPerShardId++; + if (shardRouting.relocating()) { + computeRelocatingShards++; + } + } else if (shardRouting.initializing()) { + computeInitializingShards++; + } else if (shardRouting.unassigned()) { + computeUnassignedShards++; + if (shardRouting.unassignedInfo().isDelayed()) { + computeDelayedUnassignedShards++; + } + } + } + ShardRouting primaryShard = indexShardRoutingTable.primaryShard(); + if (primaryShard.active()) { + computeActivePrimaryShards++; + } + ClusterHealthStatus shardHealth = ClusterShardHealth.getShardHealth( + primaryShard, + activeShardsPerShardId, + shardRoutingCountPerShardId + ); + computeStatus = getIndexHealthStatus(shardHealth, computeStatus); + } + } + + if (indexRoutingTable.shards() != null && indexRoutingTable.shards().isEmpty()) { + // might be since none has been created yet (two phase index creation) + computeStatus = ClusterHealthStatus.RED; + } + + this.status = computeStatus; + this.activePrimaryShards = computeActivePrimaryShards; + this.activeShards = computeActiveShards; + this.relocatingShards = computeRelocatingShards; + this.initializingShards = computeInitializingShards; + this.unassignedShards = computeUnassignedShards; + this.delayedUnassignedShards = computeDelayedUnassignedShards; + + } + + public static ClusterHealthStatus getIndexHealthStatus(ClusterHealthStatus shardHealth, ClusterHealthStatus computeStatus) { + switch (shardHealth) { + case RED: + return ClusterHealthStatus.RED; + case YELLOW: + // do not override an existing red + if (computeStatus != ClusterHealthStatus.RED) { + return ClusterHealthStatus.YELLOW; + } else { + return ClusterHealthStatus.RED; + } + default: + return computeStatus; + } } public ClusterIndexHealth(final StreamInput in) throws IOException { @@ -269,6 +373,10 @@ public int getUnassignedShards() { return unassignedShards; } + public int getDelayedUnassignedShards() { + return delayedUnassignedShards; + } + public ClusterHealthStatus getStatus() { return status; } diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java index 1fe88f65248c2..ace4537a5e291 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterShardHealth.java @@ -50,6 +50,7 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; +import java.util.List; import java.util.Locale; import java.util.Objects; @@ -109,6 +110,7 @@ public final class ClusterShardHealth implements Writeable, ToXContentFragment { private final int relocatingShards; private final int initializingShards; private final int unassignedShards; + private int delayedUnassignedShards; private final boolean primaryActive; public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable) { @@ -117,7 +119,10 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR int computeRelocatingShards = 0; int computeInitializingShards = 0; int computeUnassignedShards = 0; - for (ShardRouting shardRouting : shardRoutingTable) { + int computeDelayedUnassignedShards = 0; + List shardRoutings = shardRoutingTable.shards(); + for (int index = 0; index < shardRoutings.size(); index++) { + ShardRouting shardRouting = shardRoutings.get(index); if (shardRouting.active()) { computeActiveShards++; if (shardRouting.relocating()) { @@ -128,24 +133,18 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR computeInitializingShards++; } else if (shardRouting.unassigned()) { computeUnassignedShards++; + if (shardRouting.unassignedInfo() != null && shardRouting.unassignedInfo().isDelayed()) { + computeDelayedUnassignedShards++; + } } } - ClusterHealthStatus computeStatus; final ShardRouting primaryRouting = shardRoutingTable.primaryShard(); - if (primaryRouting.active()) { - if (computeActiveShards == shardRoutingTable.size()) { - computeStatus = ClusterHealthStatus.GREEN; - } else { - computeStatus = ClusterHealthStatus.YELLOW; - } - } else { - computeStatus = getInactivePrimaryHealth(primaryRouting); - } - this.status = computeStatus; + this.status = getShardHealth(primaryRouting, computeActiveShards, shardRoutingTable.size()); this.activeShards = computeActiveShards; this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.delayedUnassignedShards = computeDelayedUnassignedShards; this.primaryActive = primaryRouting.active(); } @@ -208,6 +207,10 @@ public int getUnassignedShards() { return unassignedShards; } + public int getDelayedUnassignedShards() { + return delayedUnassignedShards; + } + @Override public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(shardId); @@ -219,6 +222,27 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeBoolean(primaryActive); } + /** + * Computes the shard health of an index. + *

+ * Shard health is GREEN when all primary and replica shards of the indices are active. + * Shard health is YELLOW when primary shard is active but at-least one replica shard is inactive. + * Shard health is RED when the primary is not active. + *

+ */ + public static ClusterHealthStatus getShardHealth(final ShardRouting primaryRouting, final int activeShards, final int totalShards) { + assert primaryRouting != null : "Primary shard routing can't be null"; + if (primaryRouting.active()) { + if (activeShards == totalShards) { + return ClusterHealthStatus.GREEN; + } else { + return ClusterHealthStatus.YELLOW; + } + } else { + return getInactivePrimaryHealth(primaryRouting); + } + } + /** * Checks if an inactive primary shard should cause the cluster health to go RED. *

diff --git a/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java index 083159bffdc2b..f6cfdd3c42e0c 100644 --- a/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java +++ b/server/src/main/java/org/opensearch/cluster/health/ClusterStateHealth.java @@ -31,6 +31,7 @@ package org.opensearch.cluster.health; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -63,6 +64,7 @@ public final class ClusterStateHealth implements Iterable, W private final int activePrimaryShards; private final int initializingShards; private final int unassignedShards; + private int delayedUnassignedShards; private final double activeShardsPercent; private final ClusterHealthStatus status; private final Map indices; @@ -76,6 +78,10 @@ public ClusterStateHealth(final ClusterState clusterState) { this(clusterState, clusterState.metadata().getConcreteAllIndices()); } + public ClusterStateHealth(final ClusterState clusterState, final ClusterHealthRequest.Level clusterHealthLevel) { + this(clusterState, clusterState.metadata().getConcreteAllIndices(), clusterHealthLevel); + } + /** * Creates a new ClusterStateHealth instance considering the current cluster state and the provided index names. * @@ -105,6 +111,7 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre int computeRelocatingShards = 0; int computeInitializingShards = 0; int computeUnassignedShards = 0; + int computeDelayedUnassignedShards = 0; for (ClusterIndexHealth indexHealth : indices.values()) { computeActivePrimaryShards += indexHealth.getActivePrimaryShards(); @@ -112,10 +119,76 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre computeRelocatingShards += indexHealth.getRelocatingShards(); computeInitializingShards += indexHealth.getInitializingShards(); computeUnassignedShards += indexHealth.getUnassignedShards(); - if (indexHealth.getStatus() == ClusterHealthStatus.RED) { - computeStatus = ClusterHealthStatus.RED; - } else if (indexHealth.getStatus() == ClusterHealthStatus.YELLOW && computeStatus != ClusterHealthStatus.RED) { - computeStatus = ClusterHealthStatus.YELLOW; + computeDelayedUnassignedShards += indexHealth.getDelayedUnassignedShards(); + computeStatus = getClusterHealthStatus(indexHealth, computeStatus); + } + + if (clusterState.blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) { + computeStatus = ClusterHealthStatus.RED; + } + + this.status = computeStatus; + this.activePrimaryShards = computeActivePrimaryShards; + this.activeShards = computeActiveShards; + this.relocatingShards = computeRelocatingShards; + this.initializingShards = computeInitializingShards; + this.unassignedShards = computeUnassignedShards; + this.delayedUnassignedShards = computeDelayedUnassignedShards; + + // shortcut on green + if (ClusterHealthStatus.GREEN.equals(computeStatus)) { + this.activeShardsPercent = 100; + } else { + List shardRoutings = clusterState.getRoutingTable().allShards(); + int activeShardCount = 0; + int totalShardCount = 0; + for (ShardRouting shardRouting : shardRoutings) { + if (shardRouting.active()) activeShardCount++; + totalShardCount++; + } + this.activeShardsPercent = (((double) activeShardCount) / totalShardCount) * 100; + } + } + + public ClusterStateHealth( + final ClusterState clusterState, + final String[] concreteIndices, + final ClusterHealthRequest.Level healthLevel + ) { + numberOfNodes = clusterState.nodes().getSize(); + numberOfDataNodes = clusterState.nodes().getDataNodes().size(); + hasDiscoveredClusterManager = clusterState.nodes().getClusterManagerNodeId() != null; + indices = new HashMap<>(); + boolean isIndexOrShardLevelHealthRequired = healthLevel == ClusterHealthRequest.Level.INDICES + || healthLevel == ClusterHealthRequest.Level.SHARDS; + + ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN; + int computeActivePrimaryShards = 0; + int computeActiveShards = 0; + int computeRelocatingShards = 0; + int computeInitializingShards = 0; + int computeUnassignedShards = 0; + int computeDelayedUnassignedShards = 0; + + for (String index : concreteIndices) { + IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(index); + IndexMetadata indexMetadata = clusterState.metadata().index(index); + if (indexRoutingTable == null) { + continue; + } + + ClusterIndexHealth indexHealth = new ClusterIndexHealth(indexMetadata, indexRoutingTable, healthLevel); + computeActivePrimaryShards += indexHealth.getActivePrimaryShards(); + computeActiveShards += indexHealth.getActiveShards(); + computeRelocatingShards += indexHealth.getRelocatingShards(); + computeInitializingShards += indexHealth.getInitializingShards(); + computeUnassignedShards += indexHealth.getUnassignedShards(); + computeDelayedUnassignedShards += indexHealth.getDelayedUnassignedShards(); + computeStatus = getClusterHealthStatus(indexHealth, computeStatus); + + if (isIndexOrShardLevelHealthRequired) { + // Store ClusterIndexHealth only when the health is requested at Index or Shard level + indices.put(indexHealth.getIndex(), indexHealth); } } @@ -129,9 +202,10 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.delayedUnassignedShards = computeDelayedUnassignedShards; // shortcut on green - if (computeStatus.equals(ClusterHealthStatus.GREEN)) { + if (ClusterHealthStatus.GREEN.equals(computeStatus)) { this.activeShardsPercent = 100; } else { List shardRoutings = clusterState.getRoutingTable().allShards(); @@ -145,6 +219,22 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre } } + private static ClusterHealthStatus getClusterHealthStatus(ClusterIndexHealth indexHealth, ClusterHealthStatus computeStatus) { + switch (indexHealth.getStatus()) { + case RED: + return ClusterHealthStatus.RED; + case YELLOW: + // do not override an existing red + if (computeStatus != ClusterHealthStatus.RED) { + return ClusterHealthStatus.YELLOW; + } else { + return ClusterHealthStatus.RED; + } + default: + return computeStatus; + } + } + public ClusterStateHealth(final StreamInput in) throws IOException { activePrimaryShards = in.readVInt(); activeShards = in.readVInt(); @@ -213,6 +303,10 @@ public int getUnassignedShards() { return unassignedShards; } + public int getDelayedUnassignedShards() { + return delayedUnassignedShards; + } + public int getNumberOfNodes() { return this.numberOfNodes; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java index e29a81a2c131f..78f17c9ff212b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.cluster.ClusterInfoService; import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.ClusterState; @@ -196,7 +197,11 @@ public ClusterState applyStartedShards(ClusterState clusterState, List { + public void testClusterShardGreenHealth() { + String indexName = "test"; + int shardID = 0; + Index index = new Index(indexName, UUID.randomUUID().toString()); + ShardId shardId = new ShardId(index, shardID); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, "node_0", null, true, ShardRoutingState.STARTED) + ); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, "node_1", null, false, ShardRoutingState.STARTED) + ); + IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build(); + ClusterShardHealth clusterShardHealth = new ClusterShardHealth(shardID, indexShardRoutingTable); + assertEquals(2, clusterShardHealth.getActiveShards()); + assertEquals(0, clusterShardHealth.getInitializingShards()); + assertEquals(0, clusterShardHealth.getRelocatingShards()); + assertEquals(0, clusterShardHealth.getUnassignedShards()); + assertEquals(0, clusterShardHealth.getDelayedUnassignedShards()); + assertEquals(ClusterHealthStatus.GREEN, clusterShardHealth.getStatus()); + assertTrue(clusterShardHealth.isPrimaryActive()); + } + + public void testClusterShardYellowHealth() { + String indexName = "test"; + int shardID = 0; + Index index = new Index(indexName, UUID.randomUUID().toString()); + ShardId shardId = new ShardId(index, shardID); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, "node_0", null, true, ShardRoutingState.STARTED) + ); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, "node_1", "node_5", false, ShardRoutingState.RELOCATING) + ); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, "node_2", null, false, ShardRoutingState.INITIALIZING) + ); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, null, null, false, ShardRoutingState.UNASSIGNED) + ); + indexShardRoutingBuilder.addShard( + ShardRouting.newUnassigned( + shardId, + false, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo( + UnassignedInfo.Reason.NODE_LEFT, + "node_4 left", + null, + 0, + 0, + 0, + true, + UnassignedInfo.AllocationStatus.NO_ATTEMPT, + Collections.emptySet() + ) + ) + ); + IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build(); + ClusterShardHealth clusterShardHealth = new ClusterShardHealth(shardID, indexShardRoutingTable); + assertEquals(2, clusterShardHealth.getActiveShards()); + assertEquals(1, clusterShardHealth.getInitializingShards()); + assertEquals(1, clusterShardHealth.getRelocatingShards()); + assertEquals(2, clusterShardHealth.getUnassignedShards()); + assertEquals(1, clusterShardHealth.getDelayedUnassignedShards()); + assertEquals(ClusterHealthStatus.YELLOW, clusterShardHealth.getStatus()); + assertTrue(clusterShardHealth.isPrimaryActive()); + } + + public void testClusterShardRedHealth() { + String indexName = "test"; + int shardID = 0; + Index index = new Index(indexName, UUID.randomUUID().toString()); + ShardId shardId = new ShardId(index, shardID); + IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); + indexShardRoutingBuilder.addShard( + ShardRouting.newUnassigned( + shardId, + true, + RecoverySource.PeerRecoverySource.INSTANCE, + new UnassignedInfo( + UnassignedInfo.Reason.NODE_LEFT, + "node_4 left", + null, + 0, + 0, + 0, + false, + UnassignedInfo.AllocationStatus.DECIDERS_NO, + Collections.emptySet() + ) + ) + ); + indexShardRoutingBuilder.addShard( + TestShardRouting.newShardRouting(indexName, shardID, null, null, false, ShardRoutingState.UNASSIGNED) + ); + IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build(); + ClusterShardHealth clusterShardHealth = new ClusterShardHealth(shardID, indexShardRoutingTable); + assertEquals(0, clusterShardHealth.getActiveShards()); + assertEquals(0, clusterShardHealth.getInitializingShards()); + assertEquals(0, clusterShardHealth.getRelocatingShards()); + assertEquals(2, clusterShardHealth.getUnassignedShards()); + assertEquals(0, clusterShardHealth.getDelayedUnassignedShards()); + assertEquals(ClusterHealthStatus.RED, clusterShardHealth.getStatus()); + assertFalse(clusterShardHealth.isPrimaryActive()); + } + + public void testShardRoutingNullCheck() { + assertThrows(AssertionError.class, () -> ClusterShardHealth.getShardHealth(null, 0, 0)); + } + @Override protected ClusterShardHealth doParseInstance(XContentParser parser) throws IOException { return ClusterShardHealth.fromXContent(parser); diff --git a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java index 795dc8a624e38..67e2fb59b0d74 100644 --- a/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java +++ b/server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java @@ -73,6 +73,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -83,6 +84,9 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.opensearch.action.admin.cluster.health.ClusterHealthRequest.Level.CLUSTER; +import static org.opensearch.action.admin.cluster.health.ClusterHealthRequest.Level.INDICES; +import static org.opensearch.action.admin.cluster.health.ClusterHealthRequest.Level.SHARDS; import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.opensearch.test.ClusterServiceUtils.setState; import static org.hamcrest.CoreMatchers.allOf; @@ -225,7 +229,13 @@ public void testClusterHealth() throws IOException { clusterStateHealth.hasDiscoveredClusterManager(), equalTo(clusterService.state().nodes().getClusterManagerNodeId() != null) ); - assertClusterHealth(clusterStateHealth, counter); + assertClusterHealth(clusterStateHealth, counter, SHARDS); + clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices, ClusterHealthRequest.Level.CLUSTER); + assertClusterHealth(clusterStateHealth, counter, CLUSTER); + clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices, INDICES); + assertClusterHealth(clusterStateHealth, counter, INDICES); + clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices, SHARDS); + assertClusterHealth(clusterStateHealth, counter, SHARDS); } public void testClusterHealthOnIndexCreation() { @@ -586,7 +596,11 @@ private boolean primaryInactiveDueToRecovery(final String indexName, final Clust return true; } - private void assertClusterHealth(ClusterStateHealth clusterStateHealth, RoutingTableGenerator.ShardCounter counter) { + private void assertClusterHealth( + ClusterStateHealth clusterStateHealth, + RoutingTableGenerator.ShardCounter counter, + ClusterHealthRequest.Level level + ) { assertThat(clusterStateHealth.getStatus(), equalTo(counter.status())); assertThat(clusterStateHealth.getActiveShards(), equalTo(counter.active)); assertThat(clusterStateHealth.getActivePrimaryShards(), equalTo(counter.primaryActive)); @@ -594,5 +608,16 @@ private void assertClusterHealth(ClusterStateHealth clusterStateHealth, RoutingT assertThat(clusterStateHealth.getRelocatingShards(), equalTo(counter.relocating)); assertThat(clusterStateHealth.getUnassignedShards(), equalTo(counter.unassigned)); assertThat(clusterStateHealth.getActiveShardsPercent(), is(allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)))); + if (level.equals(INDICES) || level.equals(SHARDS)) { + assertTrue(clusterStateHealth.getIndices().size() > 0); + if (level.equals(SHARDS)) { + Collection clusterIndexHealths = clusterStateHealth.getIndices().values(); + for (ClusterIndexHealth clusterIndexHealth : clusterIndexHealths) { + assertFalse(clusterIndexHealth.getShards().isEmpty()); + } + } + } else { + assertTrue(clusterStateHealth.getIndices().isEmpty()); + } } } From 0f29b529a92562fb420391a8e00b00d27c66b937 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:04:02 +0530 Subject: [PATCH 079/357] Mute RemoteMigration flaky test (#15795) Signed-off-by: Lakshya Taragi --- .../remotemigration/RemoteStoreMigrationSettingsUpdateIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java index 4a6cb28b6db00..30c597e405f4e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java @@ -68,6 +68,7 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() assertRemoteStoreBackedIndex(indexName2); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793") public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception { logger.info("Initialize cluster: gives non remote cluster manager"); initializeCluster(false); From 66c7780851231e7ec5e30181457088377da26f97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 07:53:03 -0400 Subject: [PATCH 080/357] Bump org.apache.commons:commons-lang3 from 3.16.0 to 3.17.0 in /plugins/repository-hdfs (#15580) * Bump org.apache.commons:commons-lang3 in /plugins/repository-hdfs Bumps org.apache.commons:commons-lang3 from 3.16.0 to 3.17.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/commons-lang3-3.16.0.jar.sha1 | 1 - plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/commons-lang3-3.16.0.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e09dc9d192b4e..0576d6b433e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) -- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.16.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205)) +- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.17.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205), [#15580](https://github.com/opensearch-project/OpenSearch/pull/15580)) - OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998)) - Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420)) - Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997)) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index eeb5e2cd88317..15c4a1647fd38 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -76,7 +76,7 @@ dependencies { api "org.apache.commons:commons-compress:${versions.commonscompress}" api 'org.apache.commons:commons-configuration2:2.11.0' api "commons-io:commons-io:${versions.commonsio}" - api 'org.apache.commons:commons-lang3:3.16.0' + api 'org.apache.commons:commons-lang3:3.17.0' implementation 'com.google.re2j:re2j:1.7' api 'javax.servlet:servlet-api:2.5' api "org.slf4j:slf4j-api:${versions.slf4j}" diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.16.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.16.0.jar.sha1 deleted file mode 100644 index ef4f1c1fc2002..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-lang3-3.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3eb54effe40946dfb06dc5cd6c7ce4116cd51ea4 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 new file mode 100644 index 0000000000000..073922fda1dbe --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-lang3-3.17.0.jar.sha1 @@ -0,0 +1 @@ +b17d2136f0460dcc0d2016ceefca8723bdf4ee70 \ No newline at end of file From 7c9c01d8831f57b853647bebebd8d91802186778 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Sat, 7 Sep 2024 00:57:26 +0530 Subject: [PATCH 081/357] Change version to 2.17.0 for shallow snapshot V2 restore changes (#15723) Signed-off-by: Sachin Kale Co-authored-by: Sachin Kale --- .../cluster/snapshots/restore/RestoreSnapshotRequest.java | 4 ++-- .../java/org/opensearch/cluster/routing/RecoverySource.java | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 409c48cabad35..469a31407dc5d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -161,7 +161,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_10_0)) { sourceRemoteStoreRepository = in.readOptionalString(); } - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { sourceRemoteTranslogRepository = in.readOptionalString(); } } @@ -188,7 +188,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_10_0)) { out.writeOptionalString(sourceRemoteStoreRepository); } - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalString(sourceRemoteTranslogRepository); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index 43e195ed47553..ae96ea6b73cca 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -48,8 +48,6 @@ import java.io.IOException; import java.util.Objects; -import static org.opensearch.Version.CURRENT; - /** * Represents the recovery source of a shard. Available recovery types are: *

@@ -335,7 +333,7 @@ public SnapshotRecoverySource( remoteStoreIndexShallowCopy = false; sourceRemoteStoreRepository = null; } - if (in.getVersion().onOrAfter(CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { sourceRemoteTranslogRepository = in.readOptionalString(); pinnedTimestamp = in.readLong(); } else { @@ -399,7 +397,7 @@ protected void writeAdditionalFields(StreamOutput out) throws IOException { out.writeBoolean(remoteStoreIndexShallowCopy); out.writeOptionalString(sourceRemoteStoreRepository); } - if (out.getVersion().onOrAfter(CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalString(sourceRemoteTranslogRepository); out.writeLong(pinnedTimestamp); } From 767c07f46d361a991b3b6ea2afc2ac501f63b04e Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Sun, 8 Sep 2024 07:54:40 +0530 Subject: [PATCH 082/357] Downgrade version check for ser/de of applyLevelAtTransportLayer flag in ClusterHealthRequest (#15803) Signed-off-by: Swetha Guptha --- .../action/admin/cluster/health/ClusterHealthRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java index 55d3e2089b2a8..65fa32fecda2e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -115,7 +115,7 @@ public ClusterHealthRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_6_0)) { ensureNodeWeighedIn = in.readBoolean(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { applyLevelAtTransportLayer = in.readBoolean(); } } @@ -153,7 +153,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_6_0)) { out.writeBoolean(ensureNodeWeighedIn); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeBoolean(applyLevelAtTransportLayer); } } From 5642ce7b4a432e25e6d71ceab961165bcd7eaad4 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sun, 8 Sep 2024 10:31:27 -0400 Subject: [PATCH 083/357] Update protobuf from 3.22.3 to 3.25.4 (#15684) * Update protobuf from 3.22.3 to 3.25.4 Signed-off-by: Craig Perkins * Add CHANGELOG entry Signed-off-by: Craig Perkins * Remove double hyphen Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- plugins/repository-gcs/build.gradle | 3 --- server/licenses/protobuf-java-3.22.3.jar.sha1 | 1 - server/licenses/protobuf-java-3.25.4.jar.sha1 | 1 + 5 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 server/licenses/protobuf-java-3.22.3.jar.sha1 create mode 100644 server/licenses/protobuf-java-3.25.4.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0576d6b433e46..6c95f84d78842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) - Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) +- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) ### Changed - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 0f066de481ea2..0c6d0536dd1d5 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -20,7 +20,7 @@ woodstox = 6.4.0 kotlin = 1.7.10 antlr4 = 4.13.1 guava = 32.1.1-jre -protobuf = 3.22.3 +protobuf = 3.25.4 jakarta_annotation = 1.3.5 google_http_client = 1.44.1 tdigest = 3.3 diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 110df89f25de8..94d25af94e67f 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -149,9 +149,6 @@ thirdPartyAudit { 'com.google.appengine.api.urlfetch.URLFetchService', 'com.google.appengine.api.urlfetch.URLFetchServiceFactory', 'com.google.auth.oauth2.GdchCredentials', - 'com.google.protobuf.MapFieldBuilder', - 'com.google.protobuf.MapFieldBuilder$Converter', - 'com.google.protobuf.MapFieldReflectionAccessor', 'com.google.protobuf.util.JsonFormat', 'com.google.protobuf.util.JsonFormat$Parser', 'com.google.protobuf.util.JsonFormat$Printer', diff --git a/server/licenses/protobuf-java-3.22.3.jar.sha1 b/server/licenses/protobuf-java-3.22.3.jar.sha1 deleted file mode 100644 index 80feeec023e7b..0000000000000 --- a/server/licenses/protobuf-java-3.22.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fdee98b8f6abab73f146a4edb4c09e56f8278d03 \ No newline at end of file diff --git a/server/licenses/protobuf-java-3.25.4.jar.sha1 b/server/licenses/protobuf-java-3.25.4.jar.sha1 new file mode 100644 index 0000000000000..21b6fdfd24dc8 --- /dev/null +++ b/server/licenses/protobuf-java-3.25.4.jar.sha1 @@ -0,0 +1 @@ +43fcb86e4a411516c7fc681450f1516de0b862a2 \ No newline at end of file From 9b014552bf42010a4d1624506345d61ee9a23882 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 9 Sep 2024 10:54:03 -0400 Subject: [PATCH 084/357] Add 2.17 release notes (#15762) (#15768) (#15776) Signed-off-by: Andriy Redko --- CHANGELOG.md | 94 +--------------- .../opensearch.release-notes-2.17.0.md | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 93 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.17.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c95f84d78842..9d1faea226390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,54 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- [Offline Nodes] Adds offline-tasks library containing various interfaces to be used for Offline Background Tasks. ([#13574](https://github.com/opensearch-project/OpenSearch/pull/13574)) -- Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972)) -- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708)) -- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991)) -- [Workload Management] Add Delete QueryGroup API Logic ([#14735](https://github.com/opensearch-project/OpenSearch/pull/14735)) -- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437)) -- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618)) -- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680)) -- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016)) -- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039)) -- [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072)) -- Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711)) -- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) -- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709)) -- [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028)) -- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775)) -- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) -- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) -- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) -- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239)) -- Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) -- Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) -- Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) -- Support cancellation for cat shards and node stats API.([#13966](https://github.com/opensearch-project/OpenSearch/pull/13966)) -- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381)) -- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) -- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) -- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) -- Add limit on number of processors for Ingest pipeline([#15460](https://github.com/opensearch-project/OpenSearch/pull/15465)). -- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630)) -- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454)) -- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) -- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) -- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) -- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237)) -- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) -- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) -- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) -- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) -- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291))) -- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409)) -- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) -- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664)) -- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) -- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) -- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) -- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) -- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) @@ -62,62 +14,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies -- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) -- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.17.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861), [#15205](https://github.com/opensearch-project/OpenSearch/pull/15205), [#15580](https://github.com/opensearch-project/OpenSearch/pull/15580)) -- OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998)) -- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420)) -- Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997)) -- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110)) -- Bump `actions/setup-java` from 1 to 4 ([#15104](https://github.com/opensearch-project/OpenSearch/pull/15104)) -- Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119)) -- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.9 ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103), [#15299](https://github.com/opensearch-project/OpenSearch/pull/15299)) -- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111)) -- Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207)) -- Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206)) -- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) -- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) -- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301)) -- Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300)) -- Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297)) -- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298)) -- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) -- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) -- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422)) -- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418)) -- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419)) -- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) -- Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) ### Changed -- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) -- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371)) -- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238)) -- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216)) ### Deprecated ### Removed -- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518)) ### Fixed -- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908)) -- Fix NPE when bulk ingest with empty pipeline ([#15033](https://github.com/opensearch-project/OpenSearch/pull/15033)) -- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963)) -- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080)) -- Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126)) -- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620)) -- Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194)) -- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233)) -- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375)) -- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) -- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) -- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) -- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) -- Fix remote snapshot file_cache exceeding capacity ([#15077](https://github.com/opensearch-project/OpenSearch/pull/15077)) ### Security -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.15...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md new file mode 100644 index 0000000000000..18b7c9ac7cd68 --- /dev/null +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -0,0 +1,104 @@ +## 2024-09-17 Version 2.17.0 Release Notes + +## [2.17.0] +### Added +- [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028)) +- Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972)) +- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708)) +- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991)) +- [Workload Management] Add Delete QueryGroup API Logic ([#14735](https://github.com/opensearch-project/OpenSearch/pull/14735)) +- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437)) +- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618)) +- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680)) +- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016)) +- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039)) +- [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072)) +- Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711)) +- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054)) +- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709)) +- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775)) +- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897)) +- Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774)) +- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153)) +- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239)) +- Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) +- Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) +- Star tree mapping changes ([#14605](https://github.com/opensearch-project/OpenSearch/pull/14605)) +- Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) +- Support cancellation for cat shards and node stats API.([#13966](https://github.com/opensearch-project/OpenSearch/pull/13966)) +- [Streaming Indexing] Introduce bulk HTTP API streaming flavor ([#15381](https://github.com/opensearch-project/OpenSearch/pull/15381)) +- Add support for centralize snapshot creation with pinned timestamp ([#15124](https://github.com/opensearch-project/OpenSearch/pull/15124)) +- Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326)) +- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) +- Add limit on number of processors for Ingest pipeline([#15460](https://github.com/opensearch-project/OpenSearch/pull/15465)). +- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630)) +- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428))) +- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494)) +- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788)) +- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527)) +- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237)) +- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290)) +- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)) +- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426)) +- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557)) +- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010)) +- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363)) +- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568)) +- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218)) +- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454)) +- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))[SnapshotV2] Snapshot Status API changes (#15409)) +- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) +- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) +- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) + +### Dependencies +- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) +- Bump `org.apache.commons:commons-lang3` from 3.14.0 to 3.15.0 ([#14861](https://github.com/opensearch-project/OpenSearch/pull/14861)) +- OpenJDK Update (July 2024 Patch releases) ([#14998](https://github.com/opensearch-project/OpenSearch/pull/14998)) +- Bump `com.microsoft.azure:msal4j` from 1.16.1 to 1.17.0 ([#14995](https://github.com/opensearch-project/OpenSearch/pull/14995), [#15420](https://github.com/opensearch-project/OpenSearch/pull/15420)) +- Bump `actions/github-script` from 6 to 7 ([#14997](https://github.com/opensearch-project/OpenSearch/pull/14997)) +- Bump `org.tukaani:xz` from 1.9 to 1.10 ([#15110](https://github.com/opensearch-project/OpenSearch/pull/15110)) +- Bump `org.apache.avro:avro` from 1.11.3 to 1.12.0 in /plugins/repository-hdfs ([#15119](https://github.com/opensearch-project/OpenSearch/pull/15119)) +- Bump `org.bouncycastle:bcpg-fips` from 1.0.7.1 to 2.0.9 ([#15103](https://github.com/opensearch-project/OpenSearch/pull/15103), [#15299](https://github.com/opensearch-project/OpenSearch/pull/15299)) +- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207)) +- Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206)) +- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) +- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262)) +- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301)) +- Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300)) +- Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297)) +- Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298)) +- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) +- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361)) +- Bump `tj-actions/changed-files` from 44 to 45 ([#15422](https://github.com/opensearch-project/OpenSearch/pull/15422)) +- Bump `dnsjava:dnsjava` from 3.6.0 to 3.6.1 ([#15418](https://github.com/opensearch-project/OpenSearch/pull/15418)) +- Bump `com.netflix.nebula.ospackage-base` from 11.9.1 to 11.10.0 ([#15419](https://github.com/opensearch-project/OpenSearch/pull/15419)) +- Bump `org.roaringbitmap:RoaringBitmap` from 1.1.0 to 1.2.1 ([#15423](https://github.com/opensearch-project/OpenSearch/pull/15423)) +- Bump `icu4j` from 70.1 to 75.1 ([#15469](https://github.com/opensearch-project/OpenSearch/pull/15469)) + +### Changed +- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979)) +- Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371)) +- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238)) +- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216)) + +### Deprecated + +### Removed +- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518)) + +### Fixed +- Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908)) +- Fix NPE when bulk ingest with empty pipeline ([#15033](https://github.com/opensearch-project/OpenSearch/pull/15033)) +- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963)) +- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080)) +- Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126)) +- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620)) +- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233)) +- Fix range aggregation optimization ignoring top level queries ([#15287](https://github.com/opensearch-project/OpenSearch/pull/15287)) +- Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375)) +- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) +- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) +- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) +- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) From 503e6f0b76fb5feb82e0e1af35e14dea02c9fd3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:58:54 -0400 Subject: [PATCH 085/357] Bump org.apache.logging.log4j:log4j-core from 2.23.1 to 2.24.0 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#15858) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.23.1 to 2.24.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d1faea226390..c5f667a175750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) ### Changed diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 48dfb206375ca..59063b3789c70 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.23.1" + implementation "org.apache.logging.log4j:log4j-core:2.24.0" } ["0.0.1", "0.0.2"].forEach { v -> From e688d39a5d4e17c241c3415322554363312ad475 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:54:55 -0400 Subject: [PATCH 086/357] Bump peter-evans/create-pull-request from 6 to 7 (#15863) * Bump peter-evans/create-pull-request from 6 to 7 Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 6 to 7. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v6...v7) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/version.yml | 6 +++--- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 2de54716256ff..959c5c394be7d 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -62,7 +62,7 @@ jobs: - name: Create PR for BASE id: base_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: ${{ env.BASE }} branch: 'create-pull-request/patch-${{ env.BASE }}' @@ -88,7 +88,7 @@ jobs: - name: Create PR for BASE_X id: base_x_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: ${{ env.BASE_X }} branch: 'create-pull-request/patch-${{ env.BASE_X }}' @@ -114,7 +114,7 @@ jobs: - name: Create PR for main id: main_pr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: base: main branch: 'create-pull-request/patch-main' diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f667a175750..d70fc315cbf14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) +- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) ### Changed From 7cb2bd09b18da1110c74f6947c2234932a0be675 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 9 Sep 2024 11:39:15 -0700 Subject: [PATCH 087/357] Remove awaits fix since tests are now passing (#15727) Signed-off-by: Harsha Vamsi Kalluri --- .../ApproximatePointRangeQueryTests.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index 1eaea6a9e9d47..9c022aade5dc6 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -13,9 +13,12 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.store.Directory; @@ -26,6 +29,7 @@ import java.io.IOException; +import static java.util.Arrays.asList; import static org.apache.lucene.document.LongPoint.pack; import static org.mockito.Mockito.mock; @@ -112,7 +116,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeUnderDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -151,7 +154,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeWithSizeOverDefault() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -196,7 +198,6 @@ protected String toString(int dimension, byte[] value) { } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15600") public void testApproximateRangeShortCircuit() throws IOException { try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new WhitespaceAnalyzer())) { @@ -256,8 +257,7 @@ public void testApproximateRangeShortCircuitAscSort() throws IOException { for (int v = 0; v < dims; v++) { scratch[v] = i; } - doc.add(new LongPoint("point", scratch)); - iw.addDocument(doc); + iw.addDocument(asList(new LongPoint("point", scratch[0]), new NumericDocValuesField("point", scratch[0]))); } iw.flush(); iw.forceMerge(1); @@ -280,8 +280,9 @@ protected String toString(int dimension, byte[] value) { Query query = LongPoint.newRangeQuery("point", lower, upper); ; IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(approximateQuery, 10); - TopDocs topDocs1 = searcher.search(query, 10); + Sort sort = new Sort(new SortField("point", SortField.Type.LONG)); + TopDocs topDocs = searcher.search(approximateQuery, 10, sort); + TopDocs topDocs1 = searcher.search(query, 10, sort); // since we short-circuit from the approx range at the end of size these will not be equal assertNotEquals(topDocs.totalHits, topDocs1.totalHits); From 62c6d08f62309aed374ceb46d238cab52a492e0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 08:18:40 -0400 Subject: [PATCH 088/357] Bump com.nimbusds:oauth2-oidc-sdk from 11.9.1 to 11.19.1 in /plugins/repository-azure (#15862) * Bump com.nimbusds:oauth2-oidc-sdk in /plugins/repository-azure Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 11.9.1 to 11.19.1. - [Changelog](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/branches/compare/11.19.1..11.9.1) --- updated-dependencies: - dependency-name: com.nimbusds:oauth2-oidc-sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 | 1 + .../repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 | 1 - 4 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d70fc315cbf14..3a28f3852d162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) +- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index c8767478e8dad..5669c5e559ac8 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -62,7 +62,7 @@ dependencies { api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" api 'com.microsoft.azure:msal4j:1.17.0' - api 'com.nimbusds:oauth2-oidc-sdk:11.9.1' + api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.40' api 'com.nimbusds:content-type:2.3' api 'com.nimbusds:lang-tag:1.7' diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 new file mode 100644 index 0000000000000..7d83b0e8ca639 --- /dev/null +++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.19.1.jar.sha1 @@ -0,0 +1 @@ +58db85a807a56ae76baffa519772271ad5808195 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 deleted file mode 100644 index 96d9a196a172a..0000000000000 --- a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa9a2e447e2cef4dfda40a854dd7ec35624a7799 \ No newline at end of file From 19aedcd896c060db9028b2055e03d7f3e239e59c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 10 Sep 2024 18:06:01 +0530 Subject: [PATCH 089/357] Move changelog entries to release notes which are backported to 2.17 (#15880) Signed-off-by: Shivansh Arora --- CHANGELOG.md | 2 -- release-notes/opensearch.release-notes-2.17.0.md | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a28f3852d162..20e3ef809b648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) -- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) -- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) - Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md index 18b7c9ac7cd68..c1870cafea411 100644 --- a/release-notes/opensearch.release-notes-2.17.0.md +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -50,6 +50,8 @@ - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508)) - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) +- [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) +- Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 107f0ce6a3c04ea4a759a8cf980e4d23c88ab1b8 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 10 Sep 2024 14:52:42 -0400 Subject: [PATCH 090/357] Update to Gradle 8.10.1 (#15867) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 39a291b258efb..5c6205ebf24d4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15 From f8515c7bd16feb0a2ae8c6de4e01c2d70c903624 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 11 Sep 2024 00:42:18 +0530 Subject: [PATCH 091/357] [BUG] Bug fix for checksum validation for mapping metadata (#15885) * Bug fix for checksum validation for mapping metadata Signed-off-by: Himshikha Gupta --- .../cluster/metadata/IndexMetadata.java | 40 ++++++++++++++++++- .../cluster/metadata/MappingMetadata.java | 11 ++++- .../common/compress/CompressedXContent.java | 5 +++ .../cluster/metadata/IndexMetadataTests.java | 28 ++++++++++++- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index bb470ea9e4ab8..713f8c9fc332c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1278,7 +1278,7 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti out.writeByte(state.id()); writeSettingsToStream(settings, out); out.writeVLongArray(primaryTerms); - out.writeMapValues(mappings, (stream, val) -> val.writeTo(stream)); + out.writeMapValues(mappings, (stream, val) -> val.writeVerifiableTo((BufferedChecksumStreamOutput) stream)); out.writeMapValues(aliases, (stream, val) -> val.writeTo(stream)); out.writeMap(customData, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); out.writeMap( @@ -1293,6 +1293,44 @@ public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOExcepti } } + @Override + public String toString() { + return new StringBuilder().append("IndexMetadata{routingNumShards=") + .append(routingNumShards) + .append(", index=") + .append(index) + .append(", version=") + .append(version) + .append(", state=") + .append(state) + .append(", settingsVersion=") + .append(settingsVersion) + .append(", mappingVersion=") + .append(mappingVersion) + .append(", aliasesVersion=") + .append(aliasesVersion) + .append(", primaryTerms=") + .append(Arrays.toString(primaryTerms)) + .append(", aliases=") + .append(aliases) + .append(", settings=") + .append(settings) + .append(", mappings=") + .append(mappings) + .append(", customData=") + .append(customData) + .append(", inSyncAllocationIds=") + .append(inSyncAllocationIds) + .append(", rolloverInfos=") + .append(rolloverInfos) + .append(", isSystem=") + .append(isSystem) + .append(", context=") + .append(context) + .append("}") + .toString(); + } + public boolean isSystem() { return isSystem; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java index e8180613c0fa3..5856e7861d91c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MappingMetadata.java @@ -40,8 +40,10 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; @@ -60,7 +62,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class MappingMetadata extends AbstractDiffable { +public class MappingMetadata extends AbstractDiffable implements VerifiableWriteable { public static final MappingMetadata EMPTY_MAPPINGS = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Collections.emptyMap()); private final String type; @@ -164,6 +166,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(routingRequired); } + @Override + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeString(type()); + source().writeVerifiableTo(out); + out.writeBoolean(routingRequired); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java index 23fc6353dbad3..daf2ffab2d5e2 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressedXContent.java @@ -38,6 +38,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.compress.Compressor; @@ -169,6 +170,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeByteArray(bytes); } + public void writeVerifiableTo(BufferedChecksumStreamOutput out) throws IOException { + out.writeInt(crc32); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index a92a5b41e8580..92988ab7e9cba 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -199,7 +199,30 @@ public void testWriteVerifiableTo() throws IOException { ), randomNonNegativeLong() ); - + String mappings = " {\n" + + " \"_doc\": {\n" + + " \"properties\": {\n" + + " \"actiongroups\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"allowlist\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; IndexMetadata metadata1 = IndexMetadata.builder("foo") .settings( Settings.builder() @@ -220,11 +243,13 @@ public void testWriteVerifiableTo() throws IOException { .putRolloverInfo(info1) .putRolloverInfo(info2) .putInSyncAllocationIds(0, Set.of("1", "2", "3")) + .putMapping(mappings) .build(); BytesStreamOutput out = new BytesStreamOutput(); BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out); metadata1.writeVerifiableTo(checksumOut); + assertNotNull(metadata1.toString()); IndexMetadata metadata2 = IndexMetadata.builder(metadata1.getIndex().getName()) .settings( @@ -246,6 +271,7 @@ public void testWriteVerifiableTo() throws IOException { .putRolloverInfo(info2) .putRolloverInfo(info1) .putInSyncAllocationIds(0, Set.of("3", "1", "2")) + .putMapping(mappings) .build(); BytesStreamOutput out2 = new BytesStreamOutput(); From 320611a9bd1db71503ae32e9bcd54b0c54a5f0f3 Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Wed, 11 Sep 2024 04:53:28 +0800 Subject: [PATCH 092/357] fix escaped wildcard query on wildcard field (#15737) * fix escaped wildcard query on wildcard field Signed-off-by: gesong.samuel * fix format error Signed-off-by: gesong.samuel * add change log Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Signed-off-by: Michael Froh Co-authored-by: gesong.samuel Co-authored-by: Michael Froh --- CHANGELOG.md | 1 + .../index/mapper/WildcardFieldMapper.java | 54 ++++++++++++++----- .../index/mapper/WildcardFieldTypeTests.java | 26 +++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e3ef809b648..4912d465a87a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Removed ### Fixed +- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) ### Security diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index cd95e320209ee..0cb416a9b8370 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -40,7 +40,6 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; @@ -430,22 +429,27 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo finalValue = value; } Predicate matchPredicate; - if (value.contains("?")) { - Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue)); - CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton); + Automaton automaton = WildcardQuery.toAutomaton(new Term(name(), finalValue)); + CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton); + if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.SINGLE) { + // when type equals SINGLE, #compiledAutomaton.runAutomaton is null matchPredicate = s -> { if (caseInsensitive) { s = s.toLowerCase(Locale.ROOT); } - BytesRef valueBytes = BytesRefs.toBytesRef(s); - return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length); + return s.equals(finalValue); }; + } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.ALL) { + return existsQuery(context); + } else if (compiledAutomaton.type == CompiledAutomaton.AUTOMATON_TYPE.NONE) { + return new MatchNoDocsQuery("Wildcard expression matches nothing"); } else { matchPredicate = s -> { if (caseInsensitive) { s = s.toLowerCase(Locale.ROOT); } - return Regex.simpleMatch(finalValue, s); + BytesRef valueBytes = BytesRefs.toBytesRef(s); + return compiledAutomaton.runAutomaton.run(valueBytes.bytes, valueBytes.offset, valueBytes.length); }; } @@ -468,11 +472,18 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo // Package-private for testing static Set getRequiredNGrams(String value) { Set terms = new HashSet<>(); + + if (value.isEmpty()) { + return terms; + } + int pos = 0; + String rawSequence = null; String currentSequence = null; if (!value.startsWith("?") && !value.startsWith("*")) { // Can add prefix term - currentSequence = getNonWildcardSequence(value, 0); + rawSequence = getNonWildcardSequence(value, 0); + currentSequence = performEscape(rawSequence); if (currentSequence.length() == 1) { terms.add(new String(new char[] { 0, currentSequence.charAt(0) })); } else { @@ -480,10 +491,11 @@ static Set getRequiredNGrams(String value) { } } else { pos = findNonWildcardSequence(value, pos); - currentSequence = getNonWildcardSequence(value, pos); + rawSequence = getNonWildcardSequence(value, pos); } while (pos < value.length()) { - boolean isEndOfValue = pos + currentSequence.length() == value.length(); + boolean isEndOfValue = pos + rawSequence.length() == value.length(); + currentSequence = performEscape(rawSequence); if (!currentSequence.isEmpty() && currentSequence.length() < 3 && !isEndOfValue && pos > 0) { // If this is a prefix or suffix of length < 3, then we already have a longer token including the anchor. terms.add(currentSequence); @@ -502,8 +514,8 @@ static Set getRequiredNGrams(String value) { terms.add(new String(new char[] { a, b, 0 })); } } - pos = findNonWildcardSequence(value, pos + currentSequence.length()); - currentSequence = getNonWildcardSequence(value, pos); + pos = findNonWildcardSequence(value, pos + rawSequence.length()); + rawSequence = getNonWildcardSequence(value, pos); } return terms; } @@ -511,7 +523,7 @@ static Set getRequiredNGrams(String value) { private static String getNonWildcardSequence(String value, int startFrom) { for (int i = startFrom; i < value.length(); i++) { char c = value.charAt(i); - if (c == '?' || c == '*') { + if ((c == '?' || c == '*') && (i == 0 || value.charAt(i - 1) != '\\')) { return value.substring(startFrom, i); } } @@ -529,6 +541,22 @@ private static int findNonWildcardSequence(String value, int startFrom) { return value.length(); } + private static String performEscape(String str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.length(); i++) { + if (str.charAt(i) == '\\' && (i + 1) < str.length()) { + char c = str.charAt(i + 1); + if (c == '*' || c == '?') { + i++; + } + } + sb.append(str.charAt(i)); + } + assert !sb.toString().contains("\\*"); + assert !sb.toString().contains("\\?"); + return sb.toString(); + } + @Override public Query regexpQuery( String value, diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java index cd2a23cf94c37..1a813495e9033 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java @@ -88,6 +88,32 @@ public void testWildcardQuery() { ); } + public void testEscapedWildcardQuery() { + MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); + Set expectedTerms = new HashSet<>(); + expectedTerms.add(prefixAnchored("*")); + expectedTerms.add(suffixAnchored("*")); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String term : expectedTerms) { + builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER); + } + + assertEquals( + new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**\\*"), + ft.wildcardQuery("\\**\\*", null, null) + ); + + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\*"), ft.wildcardQuery("\\*", null, null)); + + expectedTerms.remove(suffixAnchored("*")); + builder = new BooleanQuery.Builder(); + for (String term : expectedTerms) { + builder.add(new TermQuery(new Term("field", term)), BooleanClause.Occur.FILTER); + } + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), "\\**"), ft.wildcardQuery("\\**", null, null)); + } + public void testMultipleWildcardsInQuery() { final String pattern = "a?cd*efg?h"; MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); From 92d7fe89755dc87cbc16e220473c7faae055afd6 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 11 Sep 2024 11:22:39 +0530 Subject: [PATCH 093/357] Add more wait time to assertBusy to fix RemoteFsTimestampAwareTranslogTests.testSimpleOperationsUpload (#15865) Signed-off-by: Sachin Kale --- .../index/translog/RemoteFsTimestampAwareTranslogTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 4c9da7e95dfa7..4ec68f7fb75b4 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -55,6 +55,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -310,7 +311,6 @@ public void testIndexDeletionWithNoPinnedTimestampButRecentFiles() throws Except } @Override - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15731") public void testSimpleOperationsUpload() throws Exception { ArrayList ops = new ArrayList<>(); @@ -366,7 +366,7 @@ public void testSimpleOperationsUpload() throws Exception { 10, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); - }); + }, 60, TimeUnit.SECONDS); } @Override From 98a9161aff122ecd290b3e8053eb271d65251ccd Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 11 Sep 2024 11:51:56 -0400 Subject: [PATCH 094/357] Fixing MacOS 13 assemble workflows (once again) (#15898) Signed-off-by: Andriy Redko --- .github/workflows/assemble.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml index b3838b8e5ae97..d90b05c323cf1 100644 --- a/.github/workflows/assemble.yml +++ b/.github/workflows/assemble.yml @@ -32,7 +32,10 @@ jobs: if: runner.os == 'macos' continue-on-error: true run: | - brew install docker colima coreutils + # Force QEMU 9.0.2 usage + curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb + brew install qemu.rb + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils gtimeout 15m colima start shell: bash - name: Run Gradle (assemble) From 9354dd92db5b2e50e0f9067dc4a3eba59282daf1 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:02:02 +0530 Subject: [PATCH 095/357] updating release-notes for 2.17 (#15884) Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 3 --- release-notes/opensearch.release-notes-2.17.0.md | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4912d465a87a8..43f35a9119c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) -- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) -- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) -- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/release-notes/opensearch.release-notes-2.17.0.md b/release-notes/opensearch.release-notes-2.17.0.md index c1870cafea411..e374b222104b0 100644 --- a/release-notes/opensearch.release-notes-2.17.0.md +++ b/release-notes/opensearch.release-notes-2.17.0.md @@ -52,6 +52,8 @@ - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131)) - [Remote Publication] Upload incremental cluster state on master re-election ([#15145](https://github.com/opensearch-project/OpenSearch/pull/15145)) - Static RemotePublication setting added, removed experimental feature flag ([#15478](https://github.com/opensearch-project/OpenSearch/pull/15478)) +- Making _cat/allocation API use indexLevelStats ([#15292](https://github.com/opensearch-project/OpenSearch/pull/15292)) +- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 58794ada3d18f1103b3382a8556eadc2a2e0e94b Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Wed, 11 Sep 2024 12:12:04 -0700 Subject: [PATCH 096/357] Add cancellation framework changes in wlm (#15651) * cancellation related Signed-off-by: Kiran Prakash * Update CHANGELOG.md Signed-off-by: Kiran Prakash * add better cancellation reason Signed-off-by: Kiran Prakash * Update DefaultTaskCancellationTests.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskSelectionStrategy.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor node level threshold Signed-off-by: Kiran Prakash * use query group task Signed-off-by: Kaushal Kumar * code clean up and refactorings Signed-off-by: Kaushal Kumar * add unit tests and fix existing ones Signed-off-by: Kaushal Kumar * uncomment the test case Signed-off-by: Kaushal Kumar * update CHANGELOG Signed-off-by: Kaushal Kumar * fix imports Signed-off-by: Kaushal Kumar * refactor and add UTs for new constructs Signed-off-by: Kaushal Kumar * fix javadocs Signed-off-by: Kaushal Kumar * remove code clutter Signed-off-by: Kaushal Kumar * change annotation version and task selection strategy Signed-off-by: Kaushal Kumar * rename a util class Signed-off-by: Kaushal Kumar * remove wrappers from resource type Signed-off-by: Kaushal Kumar * apply spotless Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * add rename changes Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * refactor changes and logical bug fix Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kiran Prakash Signed-off-by: Kaushal Kumar Signed-off-by: Ankit Jain Co-authored-by: Kiran Prakash Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../cluster/metadata/QueryGroup.java | 3 +- .../wlm/QueryGroupLevelResourceUsageView.java | 12 +- .../org/opensearch/wlm/QueryGroupTask.java | 24 +- .../java/org/opensearch/wlm/ResourceType.java | 41 +- .../wlm/WorkloadManagementSettings.java | 2 + .../MaximumResourceTaskSelectionStrategy.java | 71 +++ .../QueryGroupTaskCancellationService.java | 205 +++++++ .../cancellation/TaskSelectionStrategy.java | 28 + .../wlm/cancellation/package-info.java | 12 + .../wlm/tracker/CpuUsageCalculator.java | 38 ++ .../wlm/tracker/MemoryUsageCalculator.java | 35 ++ ...QueryGroupResourceUsageTrackerService.java | 21 +- .../wlm/tracker/ResourceUsageCalculator.java | 34 ++ ...QueryGroupLevelResourceUsageViewTests.java | 49 +- .../org/opensearch/wlm/ResourceTypeTests.java | 19 - ...mumResourceTaskSelectionStrategyTests.java | 126 ++++ ...ueryGroupTaskCancellationServiceTests.java | 541 ++++++++++++++++++ .../tracker/ResourceUsageCalculatorTests.java | 70 +++ ...ceUsageCalculatorTrackerServiceTests.java} | 63 +- 20 files changed, 1297 insertions(+), 98 deletions(-) create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java create mode 100644 server/src/main/java/org/opensearch/wlm/cancellation/package-info.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java create mode 100644 server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java rename server/src/test/java/org/opensearch/wlm/tracker/{QueryGroupResourceUsageTrackerServiceTests.java => ResourceUsageCalculatorTrackerServiceTests.java} (69%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43f35a9119c12..610ebb304ba1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) +- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java index dcd96dceb4bf1..0eeafdc8f5eed 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; @@ -41,7 +42,7 @@ * "updated_at": 4513232415 * } */ -@ExperimentalApi +@PublicApi(since = "2.18.0") public class QueryGroup extends AbstractDiffable implements ToXContentObject { public static final String _ID_STRING = "_id"; diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java index 7577c8573ec10..de213eaab64a8 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java @@ -8,8 +8,6 @@ package org.opensearch.wlm; -import org.opensearch.tasks.Task; - import java.util.List; import java.util.Map; @@ -20,11 +18,11 @@ */ public class QueryGroupLevelResourceUsageView { // resourceUsage holds the resource usage data for a QueryGroup at a point in time - private final Map resourceUsage; + private final Map resourceUsage; // activeTasks holds the list of active tasks for a QueryGroup at a point in time - private final List activeTasks; + private final List activeTasks; - public QueryGroupLevelResourceUsageView(Map resourceUsage, List activeTasks) { + public QueryGroupLevelResourceUsageView(Map resourceUsage, List activeTasks) { this.resourceUsage = resourceUsage; this.activeTasks = activeTasks; } @@ -34,7 +32,7 @@ public QueryGroupLevelResourceUsageView(Map resourceUsage, L * * @return The map of resource usage data */ - public Map getResourceUsageData() { + public Map getResourceUsageData() { return resourceUsage; } @@ -43,7 +41,7 @@ public Map getResourceUsageData() { * * @return The list of active tasks */ - public List getActiveTasks() { + public List getActiveTasks() { return activeTasks; } } diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java index 4eb413be61b72..a1cb766579d43 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.tasks.TaskId; @@ -17,6 +18,7 @@ import java.util.Map; import java.util.Optional; +import java.util.function.LongSupplier; import java.util.function.Supplier; import static org.opensearch.search.SearchService.NO_TIMEOUT; @@ -24,15 +26,17 @@ /** * Base class to define QueryGroup tasks */ +@PublicApi(since = "2.18.0") public class QueryGroupTask extends CancellableTask { private static final Logger logger = LogManager.getLogger(QueryGroupTask.class); public static final String QUERY_GROUP_ID_HEADER = "queryGroupId"; public static final Supplier DEFAULT_QUERY_GROUP_ID_SUPPLIER = () -> "DEFAULT_QUERY_GROUP"; + private final LongSupplier nanoTimeSupplier; private String queryGroupId; public QueryGroupTask(long id, String type, String action, String description, TaskId parentTaskId, Map headers) { - this(id, type, action, description, parentTaskId, headers, NO_TIMEOUT); + this(id, type, action, description, parentTaskId, headers, NO_TIMEOUT, System::nanoTime); } public QueryGroupTask( @@ -43,8 +47,22 @@ public QueryGroupTask( TaskId parentTaskId, Map headers, TimeValue cancelAfterTimeInterval + ) { + this(id, type, action, description, parentTaskId, headers, cancelAfterTimeInterval, System::nanoTime); + } + + public QueryGroupTask( + long id, + String type, + String action, + String description, + TaskId parentTaskId, + Map headers, + TimeValue cancelAfterTimeInterval, + LongSupplier nanoTimeSupplier ) { super(id, type, action, description, parentTaskId, headers, cancelAfterTimeInterval); + this.nanoTimeSupplier = nanoTimeSupplier; } /** @@ -69,6 +87,10 @@ public final void setQueryGroupId(final ThreadContext threadContext) { .orElse(DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); } + public long getElapsedTime() { + return nanoTimeSupplier.getAsLong() - getStartTimeNanos(); + } + @Override public boolean shouldCancelChildrenOnCancellation() { return false; diff --git a/server/src/main/java/org/opensearch/wlm/ResourceType.java b/server/src/main/java/org/opensearch/wlm/ResourceType.java index 2e8da4f57f36c..a560268a66853 100644 --- a/server/src/main/java/org/opensearch/wlm/ResourceType.java +++ b/server/src/main/java/org/opensearch/wlm/ResourceType.java @@ -10,8 +10,9 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.Task; +import org.opensearch.wlm.tracker.CpuUsageCalculator; +import org.opensearch.wlm.tracker.MemoryUsageCalculator; +import org.opensearch.wlm.tracker.ResourceUsageCalculator; import java.io.IOException; import java.util.List; @@ -24,19 +25,25 @@ */ @PublicApi(since = "2.17.0") public enum ResourceType { - CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU), true), - MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY), true); + CPU("cpu", true, CpuUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelCpuCancellationThreshold), + MEMORY("memory", true, MemoryUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelMemoryCancellationThreshold); private final String name; - private final Function getResourceUsage; private final boolean statsEnabled; - + private final ResourceUsageCalculator resourceUsageCalculator; + private final Function nodeLevelThresholdSupplier; private static List sortedValues = List.of(CPU, MEMORY); - ResourceType(String name, Function getResourceUsage, boolean statsEnabled) { + ResourceType( + String name, + boolean statsEnabled, + ResourceUsageCalculator resourceUsageCalculator, + Function nodeLevelThresholdSupplier + ) { this.name = name; - this.getResourceUsage = getResourceUsage; this.statsEnabled = statsEnabled; + this.resourceUsageCalculator = resourceUsageCalculator; + this.nodeLevelThresholdSupplier = nodeLevelThresholdSupplier; } /** @@ -61,20 +68,18 @@ public String getName() { return name; } - /** - * Gets the resource usage for a given resource type and task. - * - * @param task the task for which to calculate resource usage - * @return the resource usage - */ - public long getResourceUsage(Task task) { - return getResourceUsage.apply(task); - } - public boolean hasStatsEnabled() { return statsEnabled; } + public ResourceUsageCalculator getResourceUsageCalculator() { + return resourceUsageCalculator; + } + + public double getNodeLevelThreshold(WorkloadManagementSettings settings) { + return nodeLevelThresholdSupplier.apply(settings); + } + public static List getSortedValues() { return sortedValues; } diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java index b104925df77b3..b3577c1b3219d 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java @@ -8,6 +8,7 @@ package org.opensearch.wlm; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -15,6 +16,7 @@ /** * Main class to declare Workload Management related settings */ +@PublicApi(since = "2.18.0") public class WorkloadManagementSettings { private static final Double DEFAULT_NODE_LEVEL_MEMORY_REJECTION_THRESHOLD = 0.8; private static final Double DEFAULT_NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD = 0.9; diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java b/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java new file mode 100644 index 0000000000000..ffb326c07e7ac --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategy.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; + +/** + * Represents the highest resource consuming task first selection strategy. + */ +public class MaximumResourceTaskSelectionStrategy implements TaskSelectionStrategy { + + public MaximumResourceTaskSelectionStrategy() {} + + /** + * Returns a comparator that defines the sorting condition for tasks. + * This is the default implementation since the most resource consuming tasks are the likely to regress the performance. + * from resiliency point of view it makes sense to cancel them first + * + * @return The comparator + */ + private Comparator sortingCondition(ResourceType resourceType) { + return Comparator.comparingDouble(task -> resourceType.getResourceUsageCalculator().calculateTaskResourceUsage(task)); + } + + /** + * Selects tasks for cancellation based on the provided limit and resource type. + * The tasks are sorted based on the sorting condition and then selected until the accumulated resource usage reaches the limit. + * + * @param tasks The list of tasks from which to select + * @param limit The limit on the accumulated resource usage + * @param resourceType The type of resource to consider + * @return The list of selected tasks + * @throws IllegalArgumentException If the limit is less than zero + */ + public List selectTasksForCancellation(List tasks, double limit, ResourceType resourceType) { + if (limit < 0) { + throw new IllegalArgumentException("limit has to be greater than zero"); + } + if (limit < MIN_VALUE) { + return Collections.emptyList(); + } + + List sortedTasks = tasks.stream().sorted(sortingCondition(resourceType).reversed()).collect(Collectors.toList()); + + List selectedTasks = new ArrayList<>(); + double accumulated = 0; + for (QueryGroupTask task : sortedTasks) { + selectedTasks.add(task); + accumulated += resourceType.getResourceUsageCalculator().calculateTaskResourceUsage(task); + if ((accumulated - limit) > MIN_VALUE) { + break; + } + } + return selectedTasks; + } +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java new file mode 100644 index 0000000000000..a2c97c8d8635b --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java @@ -0,0 +1,205 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.tasks.TaskCancellation; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; +import org.opensearch.wlm.QueryGroupLevelResourceUsageView; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService.TRACKED_RESOURCES; + +/** + * Manages the cancellation of tasks enforced by QueryGroup thresholds on resource usage criteria. + * This class utilizes a strategy pattern through {@link MaximumResourceTaskSelectionStrategy} to identify tasks that exceed + * predefined resource usage limits and are therefore eligible for cancellation. + * + *

The cancellation process is initiated by evaluating the resource usage of each QueryGroup against its + * resource limits. Tasks that contribute to exceeding these limits are selected for cancellation based on the + * implemented task selection strategy.

+ * + *

Instances of this class are configured with a map linking QueryGroup IDs to their corresponding resource usage + * views, a set of active QueryGroups, and a task selection strategy. These components collectively facilitate the + * identification and cancellation of tasks that threaten to breach QueryGroup resource limits.

+ * + * @see MaximumResourceTaskSelectionStrategy + * @see QueryGroup + * @see ResourceType + */ +public class QueryGroupTaskCancellationService { + public static final double MIN_VALUE = 1e-9; + + private final WorkloadManagementSettings workloadManagementSettings; + private final TaskSelectionStrategy taskSelectionStrategy; + private final QueryGroupResourceUsageTrackerService resourceUsageTrackerService; + // a map of QueryGroupId to its corresponding QueryGroupLevelResourceUsageView object + Map queryGroupLevelResourceUsageViews; + private final Collection activeQueryGroups; + private final Collection deletedQueryGroups; + + public QueryGroupTaskCancellationService( + WorkloadManagementSettings workloadManagementSettings, + TaskSelectionStrategy taskSelectionStrategy, + QueryGroupResourceUsageTrackerService resourceUsageTrackerService, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { + this.workloadManagementSettings = workloadManagementSettings; + this.taskSelectionStrategy = taskSelectionStrategy; + this.resourceUsageTrackerService = resourceUsageTrackerService; + this.activeQueryGroups = activeQueryGroups; + this.deletedQueryGroups = deletedQueryGroups; + } + + /** + * Cancel tasks based on the implemented strategy. + */ + public final void cancelTasks(BooleanSupplier isNodeInDuress) { + queryGroupLevelResourceUsageViews = resourceUsageTrackerService.constructQueryGroupLevelUsageViews(); + // cancel tasks from QueryGroups that are in Enforced mode that are breaching their resource limits + cancelTasks(ResiliencyMode.ENFORCED); + // if the node is in duress, cancel tasks accordingly. + handleNodeDuress(isNodeInDuress); + } + + private void handleNodeDuress(BooleanSupplier isNodeInDuress) { + if (!isNodeInDuress.getAsBoolean()) { + return; + } + // List of tasks to be executed in order if the node is in duress + List> duressActions = List.of(v -> cancelTasksFromDeletedQueryGroups(), v -> cancelTasks(ResiliencyMode.SOFT)); + + for (Consumer duressAction : duressActions) { + if (!isNodeInDuress.getAsBoolean()) { + break; + } + duressAction.accept(null); + } + } + + private void cancelTasksFromDeletedQueryGroups() { + cancelTasks(getAllCancellableTasks(this.deletedQueryGroups)); + } + + /** + * Get all cancellable tasks from the QueryGroups. + * + * @return List of tasks that can be cancelled + */ + List getAllCancellableTasks(ResiliencyMode resiliencyMode) { + return getAllCancellableTasks( + activeQueryGroups.stream().filter(queryGroup -> queryGroup.getResiliencyMode() == resiliencyMode).collect(Collectors.toList()) + ); + } + + /** + * Get all cancellable tasks from the given QueryGroups. + * + * @return List of tasks that can be cancelled + */ + List getAllCancellableTasks(Collection queryGroups) { + List taskCancellations = new ArrayList<>(); + for (QueryGroup queryGroup : queryGroups) { + final List reasons = new ArrayList<>(); + List selectedTasks = new ArrayList<>(); + for (ResourceType resourceType : TRACKED_RESOURCES) { + // We need to consider the already selected tasks since those tasks also consumed the resources + double excessUsage = getExcessUsage(queryGroup, resourceType) - resourceType.getResourceUsageCalculator() + .calculateResourceUsage(selectedTasks); + if (excessUsage > MIN_VALUE) { + reasons.add(new TaskCancellation.Reason(generateReasonString(queryGroup, resourceType), 1)); + // TODO: We will need to add the cancellation callback for these resources for the queryGroup to reflect stats + + // Only add tasks not already added to avoid double cancellations + selectedTasks.addAll( + taskSelectionStrategy.selectTasksForCancellation(getTasksFor(queryGroup), excessUsage, resourceType) + .stream() + .filter(x -> selectedTasks.stream().noneMatch(y -> x.getId() != y.getId())) + .collect(Collectors.toList()) + ); + } + } + + if (!reasons.isEmpty()) { + taskCancellations.addAll( + selectedTasks.stream().map(task -> createTaskCancellation(task, reasons)).collect(Collectors.toList()) + ); + } + } + return taskCancellations; + } + + private String generateReasonString(QueryGroup queryGroup, ResourceType resourceType) { + final double currentUsage = getCurrentUsage(queryGroup, resourceType); + return "QueryGroup ID : " + + queryGroup.get_id() + + " breached the resource limit: (" + + currentUsage + + " > " + + queryGroup.getResourceLimits().get(resourceType) + + ") for resource type : " + + resourceType.getName(); + } + + private List getTasksFor(QueryGroup queryGroup) { + return queryGroupLevelResourceUsageViews.get(queryGroup.get_id()).getActiveTasks(); + } + + private void cancelTasks(ResiliencyMode resiliencyMode) { + cancelTasks(getAllCancellableTasks(resiliencyMode)); + } + + private void cancelTasks(List cancellableTasks) { + cancellableTasks.forEach(TaskCancellation::cancel); + } + + private TaskCancellation createTaskCancellation(CancellableTask task, List reasons) { + return new TaskCancellation(task, reasons, List.of(this::callbackOnCancel)); + } + + private double getExcessUsage(QueryGroup queryGroup, ResourceType resourceType) { + if (queryGroup.getResourceLimits().get(resourceType) == null + || !queryGroupLevelResourceUsageViews.containsKey(queryGroup.get_id())) { + return 0; + } + return getCurrentUsage(queryGroup, resourceType) - getNormalisedThreshold(queryGroup, resourceType); + } + + private double getCurrentUsage(QueryGroup queryGroup, ResourceType resourceType) { + final QueryGroupLevelResourceUsageView queryGroupResourceUsageView = queryGroupLevelResourceUsageViews.get(queryGroup.get_id()); + return queryGroupResourceUsageView.getResourceUsageData().get(resourceType); + } + + /** + * normalises configured value with respect to node level cancellation thresholds + * @param queryGroup instance + * @return normalised value with respect to node level cancellation thresholds + */ + private double getNormalisedThreshold(QueryGroup queryGroup, ResourceType resourceType) { + double nodeLevelCancellationThreshold = resourceType.getNodeLevelThreshold(workloadManagementSettings); + return queryGroup.getResourceLimits().get(resourceType) * nodeLevelCancellationThreshold; + } + + private void callbackOnCancel() { + // TODO Implement callback logic here mostly used for Stats + } +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java b/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java new file mode 100644 index 0000000000000..63fbf9b791a33 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/TaskSelectionStrategy.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.List; + +/** + * This interface exposes a method which implementations can use + */ +public interface TaskSelectionStrategy { + /** + * Determines how the tasks are selected from the list of given tasks based on resource type + * @param tasks to select from + * @param limit min cumulative resource usage sum of selected tasks + * @param resourceType + * @return list of tasks + */ + List selectTasksForCancellation(List tasks, double limit, ResourceType resourceType); +} diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java b/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java new file mode 100644 index 0000000000000..1ce7b571e9a9c --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/cancellation/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Workload management resource based cancellation artifacts + */ +package org.opensearch.wlm.cancellation; diff --git a/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java new file mode 100644 index 0000000000000..05c84cd767b1f --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/CpuUsageCalculator.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * class to help make cpu usage calculations for the query group + */ +public class CpuUsageCalculator extends ResourceUsageCalculator { + // This value should be initialised at the start time of the process and be used throughout the codebase + public static final int PROCESSOR_COUNT = Runtime.getRuntime().availableProcessors(); + public static final CpuUsageCalculator INSTANCE = new CpuUsageCalculator(); + + private CpuUsageCalculator() {} + + @Override + public double calculateResourceUsage(List tasks) { + double usage = tasks.stream().mapToDouble(this::calculateTaskResourceUsage).sum(); + + usage /= PROCESSOR_COUNT; + return usage; + } + + @Override + public double calculateTaskResourceUsage(QueryGroupTask task) { + return (1.0f * task.getTotalResourceUtilization(ResourceStats.CPU)) / task.getElapsedTime(); + } +} diff --git a/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java new file mode 100644 index 0000000000000..fb66ff47f58d0 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/MemoryUsageCalculator.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.monitor.jvm.JvmStats; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * class to help make memory usage calculations for the query group + */ +public class MemoryUsageCalculator extends ResourceUsageCalculator { + public static final long HEAP_SIZE_BYTES = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); + public static final MemoryUsageCalculator INSTANCE = new MemoryUsageCalculator(); + + private MemoryUsageCalculator() {} + + @Override + public double calculateResourceUsage(List tasks) { + return tasks.stream().mapToDouble(this::calculateTaskResourceUsage).sum(); + } + + @Override + public double calculateTaskResourceUsage(QueryGroupTask task) { + return (1.0f * task.getTotalResourceUtilization(ResourceStats.MEMORY)) / HEAP_SIZE_BYTES; + } +} diff --git a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java index 15852b5bbe6a8..b23d9ff342139 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java @@ -8,7 +8,6 @@ package org.opensearch.wlm.tracker; -import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; @@ -25,7 +24,6 @@ * This class tracks resource usage per QueryGroup */ public class QueryGroupResourceUsageTrackerService { - public static final EnumSet TRACKED_RESOURCES = EnumSet.allOf(ResourceType.class); private final TaskResourceTrackingService taskResourceTrackingService; @@ -44,19 +42,16 @@ public QueryGroupResourceUsageTrackerService(TaskResourceTrackingService taskRes * @return Map of QueryGroup views */ public Map constructQueryGroupLevelUsageViews() { - final Map> tasksByQueryGroup = getTasksGroupedByQueryGroup(); + final Map> tasksByQueryGroup = getTasksGroupedByQueryGroup(); final Map queryGroupViews = new HashMap<>(); // Iterate over each QueryGroup entry - for (Map.Entry> queryGroupEntry : tasksByQueryGroup.entrySet()) { - // Compute the QueryGroup usage - final EnumMap queryGroupUsage = new EnumMap<>(ResourceType.class); + for (Map.Entry> queryGroupEntry : tasksByQueryGroup.entrySet()) { + // Compute the QueryGroup resource usage + final Map queryGroupUsage = new EnumMap<>(ResourceType.class); for (ResourceType resourceType : TRACKED_RESOURCES) { - long queryGroupResourceUsage = 0; - for (Task task : queryGroupEntry.getValue()) { - queryGroupResourceUsage += resourceType.getResourceUsage(task); - } - queryGroupUsage.put(resourceType, queryGroupResourceUsage); + double usage = resourceType.getResourceUsageCalculator().calculateResourceUsage(queryGroupEntry.getValue()); + queryGroupUsage.put(resourceType, usage); } // Add to the QueryGroup View @@ -73,12 +68,12 @@ public Map constructQueryGroupLevelUsa * * @return Map of tasks grouped by QueryGroup */ - private Map> getTasksGroupedByQueryGroup() { + private Map> getTasksGroupedByQueryGroup() { return taskResourceTrackingService.getResourceAwareTasks() .values() .stream() .filter(QueryGroupTask.class::isInstance) .map(QueryGroupTask.class::cast) - .collect(Collectors.groupingBy(QueryGroupTask::getQueryGroupId, Collectors.mapping(task -> (Task) task, Collectors.toList()))); + .collect(Collectors.groupingBy(QueryGroupTask::getQueryGroupId, Collectors.mapping(task -> task, Collectors.toList()))); } } diff --git a/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java new file mode 100644 index 0000000000000..bc8317cbfbf92 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/ResourceUsageCalculator.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.wlm.QueryGroupTask; + +import java.util.List; + +/** + * This class is used to track query group level resource usage + */ +@PublicApi(since = "2.18.0") +public abstract class ResourceUsageCalculator { + /** + * calculates the current resource usage for the query group + * + * @param tasks list of tasks in the query group + */ + public abstract double calculateResourceUsage(List tasks); + + /** + * calculates the task level resource usage + * @param task QueryGroupTask + * @return task level resource usage + */ + public abstract double calculateTaskResourceUsage(QueryGroupTask task); +} diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java index 532bf3de95bd6..0c7eb721806d5 100644 --- a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java @@ -8,23 +8,34 @@ package org.opensearch.wlm; -import org.opensearch.action.search.SearchAction; -import org.opensearch.core.tasks.TaskId; -import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests; -import java.util.Collections; import java.util.List; import java.util.Map; +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; +import static org.opensearch.wlm.tracker.ResourceUsageCalculatorTests.createMockTaskWithResourceStats; +import static org.mockito.Mockito.mock; + public class QueryGroupLevelResourceUsageViewTests extends OpenSearchTestCase { - Map resourceUsage; - List activeTasks; + Map resourceUsage; + List activeTasks; + ResourceUsageCalculatorTrackerServiceTests.TestClock clock; + WorkloadManagementSettings settings; public void setUp() throws Exception { super.setUp(); - resourceUsage = Map.of(ResourceType.fromName("memory"), 34L, ResourceType.fromName("cpu"), 12L); - activeTasks = List.of(getRandomTask(4321)); + settings = mock(WorkloadManagementSettings.class); + clock = new ResourceUsageCalculatorTrackerServiceTests.TestClock(); + activeTasks = List.of(createMockTaskWithResourceStats(QueryGroupTask.class, 100, 200, 0, 1)); + clock.fastForwardBy(300); + double memoryUsage = 200.0 / HEAP_SIZE_BYTES; + double cpuUsage = 100.0 / (PROCESSOR_COUNT * 300.0); + + resourceUsage = Map.of(ResourceType.MEMORY, memoryUsage, ResourceType.CPU, cpuUsage); } public void testGetResourceUsageData() { @@ -32,7 +43,7 @@ public void testGetResourceUsageData() { resourceUsage, activeTasks ); - Map resourceUsageData = queryGroupLevelResourceUsageView.getResourceUsageData(); + Map resourceUsageData = queryGroupLevelResourceUsageView.getResourceUsageData(); assertTrue(assertResourceUsageData(resourceUsageData)); } @@ -41,23 +52,13 @@ public void testGetActiveTasks() { resourceUsage, activeTasks ); - List activeTasks = queryGroupLevelResourceUsageView.getActiveTasks(); + List activeTasks = queryGroupLevelResourceUsageView.getActiveTasks(); assertEquals(1, activeTasks.size()); - assertEquals(4321, activeTasks.get(0).getId()); + assertEquals(1, activeTasks.get(0).getId()); } - private boolean assertResourceUsageData(Map resourceUsageData) { - return resourceUsageData.get(ResourceType.fromName("memory")) == 34L && resourceUsageData.get(ResourceType.fromName("cpu")) == 12L; - } - - private Task getRandomTask(long id) { - return new Task( - id, - "transport", - SearchAction.NAME, - "test description", - new TaskId(randomLong() + ":" + randomLong()), - Collections.emptyMap() - ); + private boolean assertResourceUsageData(Map resourceUsageData) { + return (resourceUsageData.get(ResourceType.MEMORY) - 200.0 / HEAP_SIZE_BYTES) <= MIN_VALUE + && (resourceUsageData.get(ResourceType.CPU) - 100.0 / (300)) < MIN_VALUE; } } diff --git a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java index 737cbb37b554c..16bd8b7e66266 100644 --- a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java +++ b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java @@ -8,14 +8,8 @@ package org.opensearch.wlm; -import org.opensearch.action.search.SearchShardTask; -import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.CancellableTask; import org.opensearch.test.OpenSearchTestCase; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class ResourceTypeTests extends OpenSearchTestCase { public void testFromName() { @@ -35,17 +29,4 @@ public void testGetName() { assertEquals("cpu", ResourceType.CPU.getName()); assertEquals("memory", ResourceType.MEMORY.getName()); } - - public void testGetResourceUsage() { - SearchShardTask mockTask = createMockTask(SearchShardTask.class, 100, 200); - assertEquals(100, ResourceType.CPU.getResourceUsage(mockTask)); - assertEquals(200, ResourceType.MEMORY.getResourceUsage(mockTask)); - } - - private T createMockTask(Class type, long cpuUsage, long heapUsage) { - T task = mock(type); - when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); - when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); - return task; - } } diff --git a/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java b/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java new file mode 100644 index 0000000000000..dc79822c59c49 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/cancellation/MaximumResourceTaskSelectionStrategyTests.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchTask; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.core.tasks.resourcetracker.ResourceStatsType; +import org.opensearch.core.tasks.resourcetracker.ResourceUsageMetric; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.IntStream; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; + +public class MaximumResourceTaskSelectionStrategyTests extends OpenSearchTestCase { + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsGreaterThanZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = 50000.0 / HEAP_SIZE_BYTES; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(100); + List selectedTasks = testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation( + tasks, + reduceBy, + resourceType + ); + assertFalse(selectedTasks.isEmpty()); + boolean sortedInDescendingResourceUsage = IntStream.range(0, selectedTasks.size() - 1) + .noneMatch( + index -> ResourceType.MEMORY.getResourceUsageCalculator() + .calculateTaskResourceUsage(selectedTasks.get(index)) < ResourceType.MEMORY.getResourceUsageCalculator() + .calculateTaskResourceUsage(selectedTasks.get(index + 1)) + ); + assertTrue(sortedInDescendingResourceUsage); + assertTrue(tasksUsageMeetsThreshold(selectedTasks, reduceBy)); + } + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsLesserThanZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = -50.0 / HEAP_SIZE_BYTES; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(3); + try { + testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation(tasks, reduceBy, resourceType); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + assertEquals("limit has to be greater than zero", e.getMessage()); + } + } + + public void testSelectTasksToCancelSelectsTasksMeetingThreshold_ifReduceByIsEqualToZero() { + MaximumResourceTaskSelectionStrategy testHighestResourceConsumingTaskFirstSelectionStrategy = + new MaximumResourceTaskSelectionStrategy(); + double reduceBy = 0.0; + ResourceType resourceType = ResourceType.MEMORY; + List tasks = getListOfTasks(50); + List selectedTasks = testHighestResourceConsumingTaskFirstSelectionStrategy.selectTasksForCancellation( + tasks, + reduceBy, + resourceType + ); + assertTrue(selectedTasks.isEmpty()); + } + + private boolean tasksUsageMeetsThreshold(List selectedTasks, double threshold) { + double memory = 0; + for (QueryGroupTask task : selectedTasks) { + memory += ResourceType.MEMORY.getResourceUsageCalculator().calculateTaskResourceUsage(task); + if ((memory - threshold) > MIN_VALUE) { + return true; + } + } + return false; + } + + private List getListOfTasks(int numberOfTasks) { + List tasks = new ArrayList<>(); + + while (tasks.size() < numberOfTasks) { + long id = randomLong(); + final QueryGroupTask task = getRandomSearchTask(id); + long initial_memory = randomLongBetween(1, 100); + + ResourceUsageMetric[] initialTaskResourceMetrics = new ResourceUsageMetric[] { + new ResourceUsageMetric(ResourceStats.MEMORY, initial_memory) }; + task.startThreadResourceTracking(id, ResourceStatsType.WORKER_STATS, initialTaskResourceMetrics); + + long memory = initial_memory + randomLongBetween(1, 10000); + + ResourceUsageMetric[] taskResourceMetrics = new ResourceUsageMetric[] { + new ResourceUsageMetric(ResourceStats.MEMORY, memory), }; + task.updateThreadResourceStats(id, ResourceStatsType.WORKER_STATS, taskResourceMetrics); + task.stopThreadResourceTracking(id, ResourceStatsType.WORKER_STATS); + tasks.add(task); + } + + return tasks; + } + + private QueryGroupTask getRandomSearchTask(long id) { + return new SearchTask( + id, + "transport", + SearchAction.NAME, + () -> "test description", + new TaskId(randomLong() + ":" + randomLong()), + Collections.emptyMap() + ); + } +} diff --git a/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java new file mode 100644 index 0000000000000..f7a49235efc69 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java @@ -0,0 +1,541 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.cancellation; + +import org.opensearch.action.search.SearchAction; +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.tasks.TaskCancellation; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.MutableQueryGroupFragment; +import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; +import org.opensearch.wlm.QueryGroupLevelResourceUsageView; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests.TestClock; +import org.junit.Before; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class QueryGroupTaskCancellationServiceTests extends OpenSearchTestCase { + private static final String queryGroupId1 = "queryGroup1"; + private static final String queryGroupId2 = "queryGroup2"; + + private TestClock clock; + + private Map queryGroupLevelViews; + private Set activeQueryGroups; + private Set deletedQueryGroups; + private QueryGroupTaskCancellationService taskCancellation; + private WorkloadManagementSettings workloadManagementSettings; + private QueryGroupResourceUsageTrackerService resourceUsageTrackerService; + + @Before + public void setup() { + workloadManagementSettings = mock(WorkloadManagementSettings.class); + queryGroupLevelViews = new HashMap<>(); + activeQueryGroups = new HashSet<>(); + deletedQueryGroups = new HashSet<>(); + + clock = new TestClock(); + when(workloadManagementSettings.getNodeLevelCpuCancellationThreshold()).thenReturn(0.9); + when(workloadManagementSettings.getNodeLevelMemoryCancellationThreshold()).thenReturn(0.9); + resourceUsageTrackerService = mock(QueryGroupResourceUsageTrackerService.class); + taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + } + + public void testGetCancellableTasksFrom_setupAppropriateCancellationReasonAndScore() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + clock.fastForwardBy(1000); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(resourceType, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + assertEquals(1, cancellableTasksFrom.get(0).getReasons().get(0).getCancellationScore()); + } + + public void testGetCancellableTasksFrom_returnsTasksWhenBreachingThreshold() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(resourceType, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_returnsTasksWhenBreachingThresholdForMemory() { + ResourceType resourceType = ResourceType.MEMORY; + double cpuUsage = 0.0; + double memoryUsage = 0.11; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, resourceType, memoryUsage)); + + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_returnsNoTasksWhenNotBreachingThreshold() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.81; + double memoryUsage = 0.0; + Double threshold = 0.9; + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup1)); + assertTrue(cancellableTasksFrom.isEmpty()); + } + + public void testGetCancellableTasksFrom_filtersQueryGroupCorrectly() { + ResourceType resourceType = ResourceType.CPU; + double usage = 0.02; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + assertEquals(0, cancellableTasksFrom.size()); + } + + public void testCancelTasks_cancelsGivenTasks() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.011; + double memoryUsage = 0.011; + + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold, ResourceType.MEMORY, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> false); + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_cancelsTasksFromDeletedQueryGroups() { + ResourceType resourceType = ResourceType.CPU; + double activeQueryGroupCpuUsage = 0.01; + double activeQueryGroupMemoryUsage = 0.0; + double deletedQueryGroupCpuUsage = 0.01; + double deletedQueryGroupMemoryUsage = 0.0; + Double threshold = 0.01; + + QueryGroup activeQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup deletedQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + QueryGroupLevelResourceUsageView mockView2 = createResourceUsageViewMock( + resourceType, + deletedQueryGroupCpuUsage, + List.of(1000, 1001) + ); + + when(mockView1.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, activeQueryGroupCpuUsage, ResourceType.MEMORY, activeQueryGroupMemoryUsage) + ); + when(mockView2.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, deletedQueryGroupCpuUsage, ResourceType.MEMORY, deletedQueryGroupMemoryUsage) + ); + queryGroupLevelViews.put(queryGroupId1, mockView1); + queryGroupLevelViews.put(queryGroupId2, mockView2); + + activeQueryGroups.add(activeQueryGroup); + deletedQueryGroups.add(deletedQueryGroup); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFromDeletedQueryGroups = taskCancellation.getAllCancellableTasks(List.of(deletedQueryGroup)); + assertEquals(2, cancellableTasksFromDeletedQueryGroups.size()); + assertEquals(1000, cancellableTasksFromDeletedQueryGroups.get(0).getTask().getId()); + assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> true); + + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertTrue(cancellableTasksFromDeletedQueryGroups.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFromDeletedQueryGroups.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_does_not_cancelTasksFromDeletedQueryGroups_whenNodeNotInDuress() { + ResourceType resourceType = ResourceType.CPU; + double activeQueryGroupCpuUsage = 0.11; + double activeQueryGroupMemoryUsage = 0.0; + double deletedQueryGroupCpuUsage = 0.11; + double deletedQueryGroupMemoryUsage = 0.0; + + Double threshold = 0.01; + + QueryGroup activeQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup deletedQueryGroup = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + QueryGroupLevelResourceUsageView mockView2 = createResourceUsageViewMock( + resourceType, + deletedQueryGroupCpuUsage, + List.of(1000, 1001) + ); + + when(mockView1.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, activeQueryGroupCpuUsage, ResourceType.MEMORY, activeQueryGroupMemoryUsage) + ); + when(mockView2.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, deletedQueryGroupCpuUsage, ResourceType.MEMORY, deletedQueryGroupMemoryUsage) + ); + + queryGroupLevelViews.put(queryGroupId1, mockView1); + queryGroupLevelViews.put(queryGroupId2, mockView2); + activeQueryGroups.add(activeQueryGroup); + deletedQueryGroups.add(deletedQueryGroup); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFromDeletedQueryGroups = taskCancellation.getAllCancellableTasks(List.of(deletedQueryGroup)); + assertEquals(2, cancellableTasksFromDeletedQueryGroups.size()); + assertEquals(1000, cancellableTasksFromDeletedQueryGroups.get(0).getTask().getId()); + assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> false); + + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertFalse(cancellableTasksFromDeletedQueryGroups.get(0).getTask().isCancelled()); + assertFalse(cancellableTasksFromDeletedQueryGroups.get(1).getTask().isCancelled()); + } + + public void testCancelTasks_cancelsGivenTasks_WhenNodeInDuress() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage1 = 0.11; + double memoryUsage1 = 0.0; + double cpuUsage2 = 0.11; + double memoryUsage2 = 0.0; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroup queryGroup2 = new QueryGroup( + "testQueryGroup", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.SOFT, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView1 = createResourceUsageViewMock(); + when(mockView1.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage1, ResourceType.MEMORY, memoryUsage1)); + queryGroupLevelViews.put(queryGroupId1, mockView1); + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getActiveTasks()).thenReturn(List.of(getRandomSearchTask(5678), getRandomSearchTask(8765))); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage2, ResourceType.MEMORY, memoryUsage2)); + queryGroupLevelViews.put(queryGroupId2, mockView); + Collections.addAll(activeQueryGroups, queryGroup1, queryGroup2); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + activeQueryGroups, + deletedQueryGroups + ); + + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, cancellableTasksFrom.size()); + assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); + assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); + + List cancellableTasksFrom1 = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + assertEquals(2, cancellableTasksFrom1.size()); + assertEquals(5678, cancellableTasksFrom1.get(0).getTask().getId()); + assertEquals(8765, cancellableTasksFrom1.get(1).getTask().getId()); + + when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); + taskCancellation.cancelTasks(() -> true); + assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); + assertTrue(cancellableTasksFrom1.get(0).getTask().isCancelled()); + assertTrue(cancellableTasksFrom1.get(1).getTask().isCancelled()); + } + + public void testGetAllCancellableTasks_ReturnsNoTasksWhenNotBreachingThresholds() { + ResourceType resourceType = ResourceType.CPU; + double queryGroupCpuUsage = 0.09; + double queryGroupMemoryUsage = 0.0; + Double threshold = 0.1; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn( + Map.of(ResourceType.CPU, queryGroupCpuUsage, ResourceType.MEMORY, queryGroupMemoryUsage) + ); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertTrue(allCancellableTasks.isEmpty()); + } + + public void testGetAllCancellableTasks_ReturnsTasksWhenBreachingThresholds() { + ResourceType resourceType = ResourceType.CPU; + double cpuUsage = 0.11; + double memoryUsage = 0.0; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + when(mockView.getResourceUsageData()).thenReturn(Map.of(ResourceType.CPU, cpuUsage, ResourceType.MEMORY, memoryUsage)); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + assertEquals(2, allCancellableTasks.size()); + assertEquals(1234, allCancellableTasks.get(0).getTask().getId()); + assertEquals(4321, allCancellableTasks.get(1).getTask().getId()); + } + + public void testGetCancellableTasksFrom_doesNotReturnTasksWhenQueryGroupIdNotFound() { + ResourceType resourceType = ResourceType.CPU; + double usage = 0.11; + Double threshold = 0.01; + + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup1", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + QueryGroup queryGroup2 = new QueryGroup( + "testQueryGroup2", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(resourceType, threshold)), + 1L + ); + + QueryGroupLevelResourceUsageView mockView = createResourceUsageViewMock(); + queryGroupLevelViews.put(queryGroupId1, mockView); + activeQueryGroups.add(queryGroup1); + activeQueryGroups.add(queryGroup2); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(List.of(queryGroup2)); + assertEquals(0, cancellableTasksFrom.size()); + } + + private QueryGroupLevelResourceUsageView createResourceUsageViewMock() { + QueryGroupLevelResourceUsageView mockView = mock(QueryGroupLevelResourceUsageView.class); + when(mockView.getActiveTasks()).thenReturn(List.of(getRandomSearchTask(1234), getRandomSearchTask(4321))); + return mockView; + } + + private QueryGroupLevelResourceUsageView createResourceUsageViewMock(ResourceType resourceType, double usage, Collection ids) { + QueryGroupLevelResourceUsageView mockView = mock(QueryGroupLevelResourceUsageView.class); + when(mockView.getResourceUsageData()).thenReturn(Collections.singletonMap(resourceType, usage)); + when(mockView.getActiveTasks()).thenReturn(ids.stream().map(this::getRandomSearchTask).collect(Collectors.toList())); + return mockView; + } + + private QueryGroupTask getRandomSearchTask(long id) { + return new QueryGroupTask( + id, + "transport", + SearchAction.NAME, + "test description", + new TaskId(randomLong() + ":" + randomLong()), + Collections.emptyMap(), + null, + clock::getTime + ); + } +} diff --git a/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java new file mode 100644 index 0000000000000..21d9717a1aaca --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTests.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.core.tasks.resourcetracker.ResourceStats; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests.TestClock; + +import java.util.List; + +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ResourceUsageCalculatorTests extends OpenSearchTestCase { + + public void testQueryGroupCpuUsage() { + TestClock clock = new TestClock(); + long fastForwardTime = PROCESSOR_COUNT * 200L; + clock.fastForwardBy(fastForwardTime); + + double expectedQueryGroupCpuUsage = 1.0 / PROCESSOR_COUNT; + + QueryGroupTask mockTask = createMockTaskWithResourceStats(QueryGroupTask.class, fastForwardTime, 200, 0, 123); + when(mockTask.getElapsedTime()).thenReturn(fastForwardTime); + double actualUsage = ResourceType.CPU.getResourceUsageCalculator().calculateResourceUsage(List.of(mockTask)); + assertEquals(expectedQueryGroupCpuUsage, actualUsage, MIN_VALUE); + + double taskResourceUsage = ResourceType.CPU.getResourceUsageCalculator().calculateTaskResourceUsage(mockTask); + assertEquals(1.0, taskResourceUsage, MIN_VALUE); + } + + public void testQueryGroupMemoryUsage() { + QueryGroupTask mockTask = createMockTaskWithResourceStats(QueryGroupTask.class, 100, 200, 0, 123); + double actualMemoryUsage = ResourceType.MEMORY.getResourceUsageCalculator().calculateResourceUsage(List.of(mockTask)); + double expectedMemoryUsage = 200.0 / HEAP_SIZE_BYTES; + + assertEquals(expectedMemoryUsage, actualMemoryUsage, MIN_VALUE); + assertEquals( + 200.0 / HEAP_SIZE_BYTES, + ResourceType.MEMORY.getResourceUsageCalculator().calculateTaskResourceUsage(mockTask), + MIN_VALUE + ); + } + + public static T createMockTaskWithResourceStats( + Class type, + long cpuUsage, + long heapUsage, + long startTimeNanos, + long taskId + ) { + T task = mock(type); + when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); + when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); + when(task.getStartTimeNanos()).thenReturn(startTimeNanos); + when(task.getId()).thenReturn(taskId); + return task; + } +} diff --git a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java similarity index 69% rename from server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java rename to server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java index ca2891cb532f2..fe72bd6e710c8 100644 --- a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/tracker/ResourceUsageCalculatorTrackerServiceTests.java @@ -9,10 +9,8 @@ package org.opensearch.wlm.tracker; import org.opensearch.action.search.SearchShardTask; -import org.opensearch.action.search.SearchTask; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.test.OpenSearchTestCase; @@ -21,6 +19,7 @@ import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; import org.junit.After; import org.junit.Before; @@ -31,18 +30,38 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.wlm.QueryGroupTask.QUERY_GROUP_ID_HEADER; +import static org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService.MIN_VALUE; +import static org.opensearch.wlm.tracker.CpuUsageCalculator.PROCESSOR_COUNT; +import static org.opensearch.wlm.tracker.MemoryUsageCalculator.HEAP_SIZE_BYTES; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class QueryGroupResourceUsageTrackerServiceTests extends OpenSearchTestCase { +public class ResourceUsageCalculatorTrackerServiceTests extends OpenSearchTestCase { TestThreadPool threadPool; TaskResourceTrackingService mockTaskResourceTrackingService; QueryGroupResourceUsageTrackerService queryGroupResourceUsageTrackerService; + WorkloadManagementSettings settings; + + public static class TestClock { + long time; + + public void fastForwardBy(long nanos) { + time += nanos; + } + + public long getTime() { + return time; + } + } + + TestClock clock; @Before public void setup() { + clock = new TestClock(); + settings = mock(WorkloadManagementSettings.class); threadPool = new TestThreadPool(getTestName()); mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class); queryGroupResourceUsageTrackerService = new QueryGroupResourceUsageTrackerService(mockTaskResourceTrackingService); @@ -55,16 +74,24 @@ public void cleanup() { public void testConstructQueryGroupLevelViews_CreatesQueryGroupLevelUsageView_WhenTasksArePresent() { List queryGroupIds = List.of("queryGroup1", "queryGroup2", "queryGroup3"); + clock.fastForwardBy(2000); Map activeSearchShardTasks = createActiveSearchShardTasks(queryGroupIds); when(mockTaskResourceTrackingService.getResourceAwareTasks()).thenReturn(activeSearchShardTasks); + Map stringQueryGroupLevelResourceUsageViewMap = queryGroupResourceUsageTrackerService .constructQueryGroupLevelUsageViews(); for (String queryGroupId : queryGroupIds) { assertEquals( - 400, - (long) stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.MEMORY) + (400 * 1.0f) / HEAP_SIZE_BYTES, + stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.MEMORY), + MIN_VALUE + ); + assertEquals( + (200 * 1.0f) / (PROCESSOR_COUNT * 2000), + stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getResourceUsageData().get(ResourceType.CPU), + MIN_VALUE ); assertEquals(2, stringQueryGroupLevelResourceUsageViewMap.get(queryGroupId).getActiveTasks().size()); } @@ -78,14 +105,23 @@ public void testConstructQueryGroupLevelViews_CreatesQueryGroupLevelUsageView_Wh public void testConstructQueryGroupLevelUsageViews_WithTasksHavingDifferentResourceUsage() { Map activeSearchShardTasks = new HashMap<>(); + clock.fastForwardBy(2000); activeSearchShardTasks.put(1L, createMockTask(SearchShardTask.class, 100, 200, "queryGroup1")); activeSearchShardTasks.put(2L, createMockTask(SearchShardTask.class, 200, 400, "queryGroup1")); when(mockTaskResourceTrackingService.getResourceAwareTasks()).thenReturn(activeSearchShardTasks); - Map queryGroupViews = queryGroupResourceUsageTrackerService .constructQueryGroupLevelUsageViews(); - assertEquals(600, (long) queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.MEMORY)); + assertEquals( + (double) 600 / HEAP_SIZE_BYTES, + queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.MEMORY), + MIN_VALUE + ); + assertEquals( + ((double) 300) / (PROCESSOR_COUNT * 2000), + queryGroupViews.get("queryGroup1").getResourceUsageData().get(ResourceType.CPU), + MIN_VALUE + ); assertEquals(2, queryGroupViews.get("queryGroup1").getActiveTasks().size()); } @@ -100,19 +136,16 @@ private Map createActiveSearchShardTasks(List queryGroupIds) return activeSearchShardTasks; } - private T createMockTask(Class type, long cpuUsage, long heapUsage, String queryGroupId) { + private T createMockTask(Class type, long cpuUsage, long heapUsage, String queryGroupId) { T task = mock(type); - if (task instanceof SearchTask || task instanceof SearchShardTask) { - // Stash the current thread context to ensure that any existing context is preserved and restored after setting the query group - // ID. - try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { - threadPool.getThreadContext().putHeader(QUERY_GROUP_ID_HEADER, queryGroupId); - ((QueryGroupTask) task).setQueryGroupId(threadPool.getThreadContext()); - } + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + threadPool.getThreadContext().putHeader(QUERY_GROUP_ID_HEADER, queryGroupId); + task.setQueryGroupId(threadPool.getThreadContext()); } when(task.getTotalResourceUtilization(ResourceStats.CPU)).thenReturn(cpuUsage); when(task.getTotalResourceUtilization(ResourceStats.MEMORY)).thenReturn(heapUsage); when(task.getStartTimeNanos()).thenReturn((long) 0); + when(task.getElapsedTime()).thenReturn(clock.getTime()); AtomicBoolean isCancelled = new AtomicBoolean(false); doAnswer(invocation -> { From 2e98df39c07bb816b50032f84ded2f62cf7fd09c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 12 Sep 2024 01:28:56 +0530 Subject: [PATCH 097/357] Fixing the CHANGELOG.md by removing change published in 2.17 (#15903) Signed-off-by: Ankit Jain --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 610ebb304ba1b..add9fab2092bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Adding WithFieldName interface for QueryBuilders with fieldName ([#15705](https://github.com/opensearch-project/OpenSearch/pull/15705)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) From 12ff5ed9c79aa66c8824e1df7e124756746c8c94 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:54:18 +0530 Subject: [PATCH 098/357] using remote cluster-state as fallback (#15424) Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 1 + .../RemoteClusterStateTermVersionIT.java | 212 ++++++++++++++++++ .../state/TransportClusterStateAction.java | 6 +- .../TransportClusterManagerNodeAction.java | 59 ++++- .../term/GetTermVersionResponse.java | 20 ++ .../term/TransportGetTermVersionAction.java | 28 ++- .../cluster/coordination/Coordinator.java | 9 + .../cluster/service/ClusterApplier.java | 6 + .../service/ClusterApplierService.java | 16 +- .../cluster/service/ClusterService.java | 8 + .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 9 +- .../remote/RemoteClusterStateCache.java | 58 +++++ .../remote/RemoteClusterStateService.java | 36 ++- .../gateway/remote/RemoteManifestManager.java | 23 +- ...ransportClusterManagerNodeActionTests.java | 92 ++++++++ .../term/ClusterTermVersionIT.java | 108 ++++++++- .../coordination/NoOpClusterApplier.java | 5 + .../RemoteClusterStateServiceTests.java | 169 +++++++++++++- .../snapshots/SnapshotResiliencyTests.java | 6 +- 20 files changed, 853 insertions(+), 21 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java diff --git a/CHANGELOG.md b/CHANGELOG.md index add9fab2092bd..3268852cc99f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) +- Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java new file mode 100644 index 0000000000000..256c2ef44b078 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateTermVersionIT.java @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.state.ClusterStateAction; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.support.clustermanager.term.GetTermVersionAction; +import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.PublicationTransportHandler; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.transport.TransportResponse; +import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportService; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.hamcrest.Matchers.is; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteClusterStateTermVersionIT extends RemoteStoreBaseIntegTestCase { + private static final String INDEX_NAME = "test-index"; + private static final String INDEX_NAME_1 = "test-index-1"; + List indexRoutingPaths; + AtomicInteger indexRoutingFiles = new AtomicInteger(); + private final RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.HASHED_PREFIX; + + @Before + public void setup() { + asyncUploadMockFsRepo = false; + } + + protected Collection> nodePlugins() { + return List.of(MockTransportService.TestPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put( + RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING.getKey(), + RemoteStoreEnums.PathType.HASHED_PREFIX.toString() + ) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO) + .put(REMOTE_PUBLICATION_SETTING_KEY, true) + .build(); + } + + public void testRemoteClusterStateFallback() throws Exception { + BlobStoreRepository repository = prepareClusterAndVerifyRepository(); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + RemoteManifestManager remoteManifestManager = remoteClusterStateService.getRemoteManifestManager(); + Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + + String[] dataNodes = internalCluster().getDataNodeNames().toArray(String[]::new); + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, dataNodes[0]); + + String cm = internalCluster().getClusterManagerName(); + primaryService.addRequestHandlingBehavior( + PublicationTransportHandler.COMMIT_STATE_ACTION_NAME, + (handler, request, channel, task) -> { + // not committing the state + logger.info("ignoring the commit from cluster-manager {}", request); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + + String index = "index_1"; + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ); + logger.info("created index {}", index); + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + + addCallCountInterceptor(cm, callCounters); + + ClusterStateResponse stateResponseM = client(cm).admin().cluster().state(new ClusterStateRequest()).actionGet(); + + ClusterStateResponse stateResponseD = client(dataNodes[0]).admin().cluster().state(new ClusterStateRequest()).actionGet(); + assertEquals(stateResponseM, stateResponseD); + assertThat(callCounters.get(ClusterStateAction.NAME).get(), is(0)); + assertThat(callCounters.get(GetTermVersionAction.NAME).get(), is(1)); + + } + + public void testNoRemoteClusterStateFound() throws Exception { + BlobStoreRepository repository = prepareClusterAndVerifyRepository(); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + RemoteManifestManager remoteManifestManager = remoteClusterStateService.getRemoteManifestManager(); + Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + getClusterState().getClusterName().value(), + getClusterState().getMetadata().clusterUUID() + ); + + String[] dataNodes = internalCluster().getDataNodeNames().toArray(String[]::new); + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, dataNodes[0]); + primaryService.addRequestHandlingBehavior( + PublicationTransportHandler.COMMIT_STATE_ACTION_NAME, + (handler, request, channel, task) -> { + // not committing the state + logger.info("ignoring the commit from cluster-manager {}", request); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } + ); + + ClusterState state = internalCluster().clusterService().state(); + String cm = internalCluster().getClusterManagerName(); + MockTransportService cmservice = (MockTransportService) internalCluster().getInstance(TransportService.class, cm); + cmservice.addRequestHandlingBehavior(GetTermVersionAction.NAME, (handler, request, channel, task) -> { + channel.sendResponse( + new GetTermVersionResponse(new ClusterStateTermVersion(state.getClusterName(), state.stateUUID(), -1, -1), true) + ); + }); + + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + + addCallCountInterceptor(cm, callCounters); + + ClusterStateResponse stateResponseM = client(cm).admin().cluster().state(new ClusterStateRequest()).actionGet(); + ClusterStateResponse stateResponseD = client(dataNodes[0]).admin().cluster().state(new ClusterStateRequest()).actionGet(); + assertEquals(stateResponseM, stateResponseD); + assertThat(callCounters.get(ClusterStateAction.NAME).get(), is(1)); + assertThat(callCounters.get(GetTermVersionAction.NAME).get(), is(1)); + + } + + private void addCallCountInterceptor(String nodeName, Map callCounters) { + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName); + for (var ctrEnty : callCounters.entrySet()) { + primaryService.addRequestHandlingBehavior(ctrEnty.getKey(), (handler, request, channel, task) -> { + ctrEnty.getValue().incrementAndGet(); + logger.info("--> {} response redirect", ctrEnty.getKey()); + handler.messageReceived(request, channel, task); + }); + } + } + + private BlobStoreRepository prepareClusterAndVerifyRepository() throws Exception { + clusterSettingsSuppliedByTest = true; + Path segmentRepoPath = randomRepoPath(); + Path translogRepoPath = randomRepoPath(); + Path remoteRoutingTableRepoPath = randomRepoPath(); + Settings settings = buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + REMOTE_ROUTING_TABLE_REPO, + remoteRoutingTableRepoPath, + false + ); + prepareCluster(1, 3, INDEX_NAME, 1, 5, settings); + ensureGreen(INDEX_NAME); + + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REMOTE_ROUTING_TABLE_REPO); + + return repository; + } + +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java index 13ea7eaa43bf8..da91d273f3bce 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -46,10 +46,12 @@ import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -80,7 +82,8 @@ public TransportClusterStateAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + @Nullable RemoteClusterStateService remoteClusterStateService ) { super( ClusterStateAction.NAME, @@ -93,6 +96,7 @@ public TransportClusterStateAction( indexNameExpressionResolver ); this.localExecuteSupported = true; + this.remoteClusterStateService = remoteClusterStateService; } @Override diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java index 4e869f29878cd..819e09312a0df 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java @@ -49,6 +49,7 @@ import org.opensearch.cluster.ClusterStateObserver; import org.opensearch.cluster.NotClusterManagerException; import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; import org.opensearch.cluster.coordination.FailedToCommitClusterStateException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException; @@ -63,6 +64,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; import org.opensearch.tasks.Task; @@ -74,6 +77,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Predicate; @@ -95,6 +99,8 @@ public abstract class TransportClusterManagerNodeAction clusterMetadataManifest = remoteClusterStateService + .getClusterMetadataManifestByTermVersion( + clusterStateTermVersion.getClusterName().value(), + clusterStateTermVersion.getClusterUUID(), + clusterStateTermVersion.getTerm(), + clusterStateTermVersion.getVersion() + ); + if (clusterMetadataManifest.isEmpty()) { + logger.trace("could not find manifest in remote-store for ClusterStateTermVersion {}", termVersion); + return null; + } + ClusterState clusterStateFromRemote = remoteClusterStateService.getClusterStateForManifest( + appliedState.getClusterName().value(), + clusterMetadataManifest.get(), + appliedState.nodes().getLocalNode().getId(), + true + ); + + if (clusterStateFromRemote != null) { + logger.trace("Using the remote cluster-state fetched from local node, ClusterStateTermVersion {}", termVersion); + return clusterStateFromRemote; + } + } catch (Exception e) { + logger.trace("Error while fetching from remote cluster state", e); + } + } + return null; + } + private boolean checkForBlock(Request request, ClusterState localClusterState) { final ClusterBlockException blockException = checkBlock(request, localClusterState); if (blockException != null) { diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java index 0906abe57d547..cf8e6085d4b32 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java @@ -8,6 +8,7 @@ package org.opensearch.action.support.clustermanager.term; +import org.opensearch.Version; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.ClusterStateTermVersion; import org.opensearch.core.action.ActionResponse; @@ -25,18 +26,34 @@ public class GetTermVersionResponse extends ActionResponse { private final ClusterStateTermVersion clusterStateTermVersion; + private final boolean isStatePresentInRemote; + public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion) { this.clusterStateTermVersion = clusterStateTermVersion; + this.isStatePresentInRemote = false; + } + + public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion, boolean canDownloadFromRemote) { + this.clusterStateTermVersion = clusterStateTermVersion; + this.isStatePresentInRemote = canDownloadFromRemote; } public GetTermVersionResponse(StreamInput in) throws IOException { super(in); this.clusterStateTermVersion = new ClusterStateTermVersion(in); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.isStatePresentInRemote = in.readOptionalBoolean(); + } else { + this.isStatePresentInRemote = false; + } } @Override public void writeTo(StreamOutput out) throws IOException { clusterStateTermVersion.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(isStatePresentInRemote); + } } public ClusterStateTermVersion getClusterStateTermVersion() { @@ -47,4 +64,7 @@ public boolean matches(ClusterState clusterState) { return clusterStateTermVersion != null && clusterStateTermVersion.equals(new ClusterStateTermVersion(clusterState)); } + public boolean isStatePresentInRemote() { + return isStatePresentInRemote; + } } diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java index 4752a99c910e4..1cab739a20838 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java @@ -15,11 +15,14 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.discovery.Discovery; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -34,13 +37,18 @@ public class TransportGetTermVersionAction extends TransportClusterManagerNodeRe private final Logger logger = LogManager.getLogger(getClass()); + private final Discovery discovery; + + private boolean usePreCommitState = false; + @Inject public TransportGetTermVersionAction( TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + Discovery discovery ) { super( GetTermVersionAction.NAME, @@ -52,6 +60,8 @@ public TransportGetTermVersionAction( GetTermVersionRequest::new, indexNameExpressionResolver ); + this.usePreCommitState = FeatureFlags.isEnabled(FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING); + this.discovery = discovery; } @Override @@ -76,10 +86,22 @@ protected void clusterManagerOperation( ClusterState state, ActionListener listener ) throws Exception { - ActionListener.completeWith(listener, () -> buildResponse(request, state)); + if (usePreCommitState) { + ActionListener.completeWith(listener, () -> buildResponse(request, clusterService.preCommitState())); + } else { + ActionListener.completeWith(listener, () -> buildResponse(request, state)); + } + } private GetTermVersionResponse buildResponse(GetTermVersionRequest request, ClusterState state) { - return new GetTermVersionResponse(new ClusterStateTermVersion(state)); + ClusterStateTermVersion termVersion = new ClusterStateTermVersion(state); + if (discovery instanceof Coordinator) { + Coordinator coordinator = (Coordinator) discovery; + if (coordinator.isRemotePublicationEnabled()) { + return new GetTermVersionResponse(termVersion, coordinator.isRemotePublicationEnabled()); + } + } + return new GetTermVersionResponse(termVersion); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 9aaaa77bcbb23..13a57d93f03f0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -386,6 +386,8 @@ private void handleApplyCommit(ApplyCommitRequest applyCommitRequest, ActionList coordinationState.get().handleCommit(applyCommitRequest); final ClusterState committedState = hideStateIfNotRecovered(coordinationState.get().getLastAcceptedState()); applierState = mode == Mode.CANDIDATE ? clusterStateWithNoClusterManagerBlock(committedState) : committedState; + clusterApplier.setPreCommitState(applierState); + if (applyCommitRequest.getSourceNode().equals(getLocalNode())) { // cluster-manager node applies the committed state at the end of the publication process, not here. applyListener.onResponse(null); @@ -1862,4 +1864,11 @@ protected void sendApplyCommit( public static boolean isZen1Node(DiscoveryNode discoveryNode) { return Booleans.isTrue(discoveryNode.getAttributes().getOrDefault("zen1", "false")); } + + public boolean isRemotePublicationEnabled() { + if (coordinationState.get() != null) { + return coordinationState.get().isRemotePublicationEnabled(); + } + return false; + } } diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java index 5b3f7f1001779..df31b0e94d734 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplier.java @@ -49,6 +49,12 @@ public interface ClusterApplier { */ void setInitialState(ClusterState initialState); + /** + * Sets the pre-commit state for the applier. + * @param clusterState state that has been committed by coordinator to store + */ + void setPreCommitState(ClusterState clusterState); + /** * Method to invoke when a new cluster state is available to be applied * diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index b2548a8976c73..47080cfbde692 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -119,7 +119,7 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements private final Collection clusterStateListeners = new CopyOnWriteArrayList<>(); private final Map timeoutClusterStateListeners = new ConcurrentHashMap<>(); - + private final AtomicReference preCommitState = new AtomicReference<>(); // last state which is yet to be applied private final AtomicReference state; // last applied state private final String nodeName; @@ -750,4 +750,18 @@ protected long currentTimeInMillis() { protected boolean applicationMayFail() { return false; } + + /** + * Pre-commit State of the cluster-applier + * @return ClusterState + */ + public ClusterState preCommitState() { + return preCommitState.get(); + } + + @Override + public void setPreCommitState(ClusterState clusterState) { + preCommitState.set(clusterState); + } + } diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java index c3c48dd8b87ef..1a79161d223e2 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java @@ -183,6 +183,14 @@ public ClusterState state() { return clusterApplierService.state(); } + /** + * The state that is persisted to store but may not be applied to cluster. + * @return ClusterState + */ + public ClusterState preCommitState() { + return clusterApplierService.preCommitState(); + } + /** * Adds a high priority applier of updated cluster states. */ diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index c8d00f65bda10..73375f9eaa413 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,6 +39,7 @@ protected FeatureFlagSettings( FeatureFlags.PLUGGABLE_CACHE_SETTING, FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, - FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING + FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, + FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 49ecbb0a7069d..a5acea004c3b2 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -128,6 +128,12 @@ public class FeatureFlags { false, Property.NodeScope ); + public static final String TERM_VERSION_PRECOMMIT_ENABLE = "opensearch.experimental.optimization.termversion.precommit.enabled"; + public static final Setting TERM_VERSION_PRECOMMIT_ENABLE_SETTING = Setting.boolSetting( + TERM_VERSION_PRECOMMIT_ENABLE, + false, + Property.NodeScope + ); private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, @@ -139,7 +145,8 @@ public class FeatureFlags { PLUGGABLE_CACHE_SETTING, STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, - READER_WRITER_SPLIT_EXPERIMENTAL_SETTING + READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, + TERM_VERSION_PRECOMMIT_ENABLE_SETTING ); /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java new file mode 100644 index 0000000000000..de36ac4429302 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCache.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.common.collect.Tuple; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * Cache to Remote Cluster State based on term-version check. The current implementation + * caches the last highest version of cluster-state that was downloaded from cache. + * + * @opensearch.internal + */ +public class RemoteClusterStateCache { + + private final AtomicReference> clusterStateFromCache = new AtomicReference<>(); + + public ClusterState getState(String clusterName, ClusterMetadataManifest manifest) { + Tuple cache = clusterStateFromCache.get(); + if (cache != null) { + ClusterStateTermVersion manifestStateTermVersion = new ClusterStateTermVersion( + new ClusterName(clusterName), + manifest.getClusterUUID(), + manifest.getClusterTerm(), + manifest.getStateVersion() + ); + if (cache.v1().equals(manifestStateTermVersion)) { + return cache.v2(); + } + } + return null; + } + + public void putState(final ClusterState newState) { + if (newState.metadata() == null || newState.coordinationMetadata() == null) { + // ensure the remote cluster state has coordination metadata set + return; + } + + ClusterStateTermVersion cacheStateTermVersion = new ClusterStateTermVersion( + new ClusterName(newState.getClusterName().value()), + newState.metadata().clusterUUID(), + newState.term(), + newState.version() + ); + clusterStateFromCache.set(new Tuple<>(cacheStateTermVersion, newState)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e504c5abb46d3..12d10fd908b44 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -33,6 +33,7 @@ import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; @@ -117,6 +118,7 @@ * * @opensearch.internal */ +@InternalApi public class RemoteClusterStateService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -233,6 +235,7 @@ public static RemoteClusterStateValidationMode parseString(String mode) { private final boolean isPublicationEnabled; private final String remotePathPrefix; + private final RemoteClusterStateCache remoteClusterStateCache; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. public static final ToXContent.Params FORMAT_PARAMS; @@ -282,6 +285,7 @@ public RemoteClusterStateService( ClusterName.CLUSTER_NAME_SETTING.get(settings).value() ); remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService, remoteRoutingTableService); + remoteClusterStateCache = new RemoteClusterStateCache(); } /** @@ -1032,6 +1036,15 @@ public ClusterMetadataManifest getClusterMetadataManifestByFileName(String clust return remoteManifestManager.getRemoteClusterMetadataManifestByFileName(clusterUUID, fileName); } + public Optional getClusterMetadataManifestByTermVersion( + String clusterName, + String clusterUUID, + long term, + long version + ) { + return remoteManifestManager.getClusterMetadataManifestByTermVersion(clusterName, clusterUUID, term, version); + } + @Override public void close() throws IOException { remoteClusterStateCleanupManager.close(); @@ -1442,6 +1455,11 @@ public ClusterState getClusterStateForManifest( String localNodeId, boolean includeEphemeral ) throws IOException { + ClusterState stateFromCache = remoteClusterStateCache.getState(clusterName, manifest); + if (stateFromCache != null) { + return stateFromCache; + } + final ClusterState clusterState; final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (manifest.onOrAfterCodecVersion(CODEC_V2)) { @@ -1498,7 +1516,10 @@ public ClusterState getClusterStateForManifest( final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateFullDownloadSucceeded(); remoteStateStats.stateFullDownloadTook(durationMillis); - + if (includeEphemeral) { + // cache only if the entire cluster-state is present + remoteClusterStateCache.putState(clusterState); + } return clusterState; } @@ -1506,6 +1527,8 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); ClusterStateDiffManifest diff = manifest.getDiffManifest(); + boolean includeEphemeral = true; + List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { Optional uploadedIndexMetadataOptional = manifest.getIndices() .stream() @@ -1554,7 +1577,7 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C manifest.getDiffManifest() != null && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), - true + includeEphemeral ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); @@ -1588,7 +1611,6 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C .metadata(metadataBuilder) .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) .build(); - if (!remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) && manifest.getClusterStateChecksum() != null) { validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); } @@ -1596,6 +1618,9 @@ public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, C remoteStateStats.stateDiffDownloadSucceeded(); remoteStateStats.stateDiffDownloadTook(durationMillis); + assert includeEphemeral == true; + // newState includes all the fields of cluster-state (includeEphemeral=true always) + remoteClusterStateCache.putState(clusterState); return clusterState; } @@ -1989,4 +2014,9 @@ public void fullDownloadFailed() { public void diffDownloadFailed() { remoteStateStats.stateDiffDownloadFailed(); } + + RemoteClusterStateCache getRemoteClusterStateCache() { + return remoteClusterStateCache; + } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 47c847b5dc32a..b243269fe323e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -194,6 +194,17 @@ public Optional getLatestClusterMetadataManifest(String return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + public Optional getClusterMetadataManifestByTermVersion( + String clusterName, + String clusterUUID, + long term, + long version + ) { + String prefix = RemoteManifestManager.getManifestFilePrefixForTermVersion(term, version); + Optional latestManifestFileName = getManifestFileNameByPrefix(clusterName, clusterUUID, prefix); + return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); + } + public ClusterMetadataManifest getRemoteClusterMetadataManifestByFileName(String clusterUUID, String filename) throws IllegalStateException { try { @@ -293,7 +304,7 @@ private List getManifestFileNames(String clusterName, String clust } } - static String getManifestFilePrefixForTermVersion(long term, long version) { + public static String getManifestFilePrefixForTermVersion(long term, long version) { return String.join( DELIMITER, RemoteClusterMetadataManifest.MANIFEST, @@ -322,4 +333,14 @@ private Optional getLatestManifestFileName(String clusterName, String cl logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); } + + private Optional getManifestFileNameByPrefix(String clusterName, String clusterUUID, String filePrefix) + throws IllegalStateException { + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } + logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); + return Optional.empty(); + } } diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index 3c1c84653b384..00198364fc8d7 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -21,6 +21,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.ThreadedActionListener; +import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse; import org.opensearch.action.support.replication.ClusterStateCreationUtils; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -30,6 +31,8 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.coordination.FailedToCommitClusterStateException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; @@ -54,6 +57,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.rest.RestStatus; import org.opensearch.discovery.ClusterManagerNotDiscoveredException; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.NodeClosedException; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.snapshots.EmptySnapshotsInfoService; @@ -77,6 +82,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; @@ -84,8 +90,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.mockito.Mockito; + import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdaterTests.createIndexMetadataWithRemoteStoreSettings; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; @@ -94,6 +103,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; public class TransportClusterManagerNodeActionTests extends OpenSearchTestCase { private static ThreadPool threadPool; @@ -209,6 +220,8 @@ public void writeTo(StreamOutput out) throws IOException { } class Action extends TransportClusterManagerNodeAction { + private boolean localExecuteSupported = false; + Action(String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) { super( actionName, @@ -221,6 +234,18 @@ class Action extends TransportClusterManagerNodeAction { ); } + Action( + String actionName, + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + RemoteClusterStateService clusterStateService + ) { + this(actionName, transportService, clusterService, threadPool); + this.remoteClusterStateService = clusterStateService; + this.localExecuteSupported = true; + } + @Override protected void doExecute(Task task, final Request request, ActionListener listener) { // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER @@ -247,6 +272,10 @@ protected void clusterManagerOperation(Request request, ClusterState state, Acti protected ClusterBlockException checkBlock(Request request, ClusterState state) { return null; // default implementation, overridden in specific tests } + + public boolean localExecuteSupportedByAction() { + return localExecuteSupported; + } } public void testLocalOperationWithoutBlocks() throws ExecutionException, InterruptedException { @@ -715,6 +744,69 @@ protected void masterOperation(Task task, Request request, ClusterState state, A assertFalse(exception.get()); } + public void testFetchFromRemoteStore() throws InterruptedException, BrokenBarrierException, ExecutionException, IOException { + Map attributes = new HashMap<>(); + attributes.put(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "repo1"); + attributes.put(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "repo2"); + + localNode = new DiscoveryNode( + "local_node", + buildNewFakeTransportAddress(), + attributes, + Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE), + Version.CURRENT + ); + remoteNode = new DiscoveryNode( + "remote_node", + buildNewFakeTransportAddress(), + attributes, + Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE), + Version.CURRENT + ); + allNodes = new DiscoveryNode[] { localNode, remoteNode }; + + setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); + + ClusterState state = clusterService.state(); + RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); + ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(state.term() + 1) + .stateVersion(state.version() + 1) + .build(); + when( + remoteClusterStateService.getClusterMetadataManifestByTermVersion( + eq(state.getClusterName().value()), + eq(state.metadata().clusterUUID()), + eq(state.term() + 1), + eq(state.version() + 1) + ) + ).thenReturn(Optional.of(manifest)); + when(remoteClusterStateService.getClusterStateForManifest(state.getClusterName().value(), manifest, localNode.getId(), true)) + .thenReturn(buildClusterState(state, state.term() + 1, state.version() + 1)); + + PlainActionFuture listener = new PlainActionFuture<>(); + Request request = new Request(); + Action action = new Action("internal:testAction", transportService, clusterService, threadPool, remoteClusterStateService); + action.execute(request, listener); + + CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0]; + // mismatch term and version + GetTermVersionResponse termResp = new GetTermVersionResponse( + new ClusterStateTermVersion(state.getClusterName(), state.metadata().clusterUUID(), state.term() + 1, state.version() + 1), + true + ); + transport.handleResponse(capturedRequest.requestId, termResp); + // no more transport calls + assertThat(transport.capturedRequests().length, equalTo(1)); + assertTrue(listener.isDone()); + } + + private ClusterState buildClusterState(ClusterState state, long term, long version) { + CoordinationMetadata.Builder coordMetadataBuilder = CoordinationMetadata.builder().term(term); + Metadata newMetadata = Metadata.builder().coordinationMetadata(coordMetadataBuilder.build()).build(); + return ClusterState.builder(state).version(version).metadata(newMetadata).build(); + } + public void testDontAllowSwitchingToStrictCompatibilityModeForMixedCluster() { // request to change cluster compatibility mode to STRICT Settings currentCompatibilityModeSettings = Settings.builder() diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java index 7b783e025a575..7ab9da231896c 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java @@ -11,9 +11,19 @@ import org.opensearch.action.admin.cluster.state.ClusterStateAction; import org.opensearch.action.admin.cluster.state.ClusterStateRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterStateApplier; import org.opensearch.cluster.coordination.ClusterStateTermVersion; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.mapper.MapperService; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -22,6 +32,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.is; @@ -100,12 +111,107 @@ public void testDatanodeOutOfSync() throws Exception { assertThat(stateResponse.getState().nodes().getSize(), is(internalCluster().getNodeNames().length)); } + public void testDatanodeWithSlowClusterApplierFallbackToPublish() throws Exception { + List masters = internalCluster().startClusterManagerOnlyNodes( + 3, + Settings.builder().put(FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE, "true").build() + ); + List datas = internalCluster().startDataOnlyNodes(3); + + Map callCounters = Map.ofEntries( + Map.entry(ClusterStateAction.NAME, new AtomicInteger()), + Map.entry(GetTermVersionAction.NAME, new AtomicInteger()) + ); + ensureGreen(); + + String master = internalCluster().getClusterManagerName(); + + AtomicBoolean processState = new AtomicBoolean(); + ClusterService cmClsService = internalCluster().getInstance(ClusterService.class, datas.get(0)); + cmClsService.addStateApplier(new ClusterStateApplier() { + @Override + public void applyClusterState(ClusterChangedEvent event) { + logger.info("Slow applier started"); + while (processState.get()) { + try { + logger.info("Sleeping for 1s"); + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + logger.info("Slow applier ended"); + } + }); + + ensureGreen(); + + GetTermVersionResponse respBeforeUpdate = internalCluster().getInstance(Client.class, master) + .execute(GetTermVersionAction.INSTANCE, new GetTermVersionRequest()) + .get(); + + processState.set(true); + String index = "index_1"; + ActionFuture startCreateIndex1 = prepareCreate(index).setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ).execute(); + + ActionFuture startCreateIndex2 = prepareCreate("index_2").setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE) + .build() + ).execute(); + + // wait for cluster-manager to publish new state + waitUntil(() -> { + try { + // node is yet to ack commit to cluster-manager , only the state-update corresponding to index_1 should have been published + GetTermVersionResponse respAfterUpdate = internalCluster().getInstance(Client.class, master) + .execute(GetTermVersionAction.INSTANCE, new GetTermVersionRequest()) + .get(); + logger.info( + "data has latest , {} , {}", + respAfterUpdate.getClusterStateTermVersion().getTerm(), + respAfterUpdate.getClusterStateTermVersion().getVersion() + ); + return respBeforeUpdate.getClusterStateTermVersion().getVersion() + 1 == respAfterUpdate.getClusterStateTermVersion() + .getVersion(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + addCallCountInterceptor(master, callCounters); + ClusterStateResponse stateResponseD = internalCluster().getInstance(Client.class, datas.get(0)) + .admin() + .cluster() + .state(new ClusterStateRequest()) + .actionGet(); + logger.info("data has the version , {} , {}", stateResponseD.getState().term(), stateResponseD.getState().version()); + assertTrue(respBeforeUpdate.getClusterStateTermVersion().getVersion() + 1 == stateResponseD.getState().version()); + + processState.set(false); + + AtomicInteger clusterStateCallsOnMaster = callCounters.get(ClusterStateAction.NAME); + AtomicInteger termCallsOnMaster = callCounters.get(GetTermVersionAction.NAME); + startCreateIndex1.get(); + startCreateIndex2.get(); + assertThat(clusterStateCallsOnMaster.get(), is(0)); + assertThat(termCallsOnMaster.get(), is(1)); + } + private void addCallCountInterceptor(String nodeName, Map callCounters) { MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeName); for (var ctrEnty : callCounters.entrySet()) { primaryService.addRequestHandlingBehavior(ctrEnty.getKey(), (handler, request, channel, task) -> { ctrEnty.getValue().incrementAndGet(); - logger.info("--> {} response redirect", ClusterStateAction.NAME); + logger.info("--> {} response redirect", ctrEnty.getKey()); handler.messageReceived(request, channel, task); }); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java b/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java index 9b865ace3b082..6e0d86540cf99 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NoOpClusterApplier.java @@ -42,6 +42,11 @@ public void setInitialState(ClusterState initialState) { } + @Override + public void setPreCommitState(ClusterState clusterState) { + + } + @Override public void onNewClusterState(String source, Supplier clusterStateSupplier, ClusterApplyListener listener) { listener.onSuccess(source); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e875b1c5dc64e..b11d5e48bec06 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -46,7 +46,6 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -62,6 +61,7 @@ import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.indices.DefaultRemoteStoreSettings; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; @@ -2322,6 +2322,73 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE ); } + public void testReadLatestClusterStateFromCache() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(12) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) + .build(); + + Metadata expectedMetadata = Metadata.builder() + .clusterUUID("cluster-uuid") + .version(12) + .coordinationMetadata(CoordinationMetadata.builder().term(1).build()) + .persistentSettings(Settings.builder().put("readonly", true).build()) + .build(); + mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expectedMetadata); + + ClusterState state = remoteClusterStateService.getLatestClusterState( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID(), + true + ); + + ClusterState stateFromCache = remoteClusterStateService.getRemoteClusterStateCache() + .getState(clusterState.getClusterName().value(), expectedManifest); + assertEquals(stateFromCache.getMetadata(), state.getMetadata()); + + final ClusterMetadataManifest notExistMetadata = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(13) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID("prev-cluster-uuid") + .routingTableVersion(1) + .indicesRouting(List.of()) + .build(); + ClusterState notInCacheState = remoteClusterStateService.getRemoteClusterStateCache() + .getState(clusterState.getClusterName().value(), notExistMetadata); + assertNull(notInCacheState); + } + public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOException() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2"); @@ -2388,9 +2455,46 @@ public void testReadLatestMetadataManifestSuccess() throws IOException { assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); } + public void testReadLatestMetadataManifestSuccessByTermVersion() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); + final List indices = List.of(uploadedIndexMetadata); + + final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .indices(indices) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID("state-uuid") + .clusterUUID("cluster-uuid") + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .codecVersion(CODEC_V2) + .previousClusterUUID("prev-cluster-uuid") + .build(); + + mockBlobContainer(mockBlobStoreObjects(), expectedManifest, new HashMap<>(), CODEC_V2, 1, 1); + remoteClusterStateService.start(); + final ClusterMetadataManifest manifest = remoteClusterStateService.getClusterMetadataManifestByTermVersion( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID(), + 1, + 1 + ).get(); + + assertThat(manifest.getIndices().size(), is(1)); + assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); + assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); + assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); + assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); + assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); + assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); + assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + } + public void testReadGlobalMetadata() throws IOException { - when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry( - List.of(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexGraveyard.TYPE), IndexGraveyard::fromXContent)))); + // when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(new NamedXContentRegistry( + // List.of(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(IndexGraveyard.TYPE), + // IndexGraveyard::fromXContent)))); final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); @@ -2405,7 +2509,10 @@ public void testReadGlobalMetadata() throws IOException { .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) - .put(IndexGraveyard.TYPE, new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) + .put( + IndexGraveyard.TYPE, + new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" + IndexGraveyard.TYPE + "-file") + ) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") @@ -2413,7 +2520,10 @@ public void testReadGlobalMetadata() throws IOException { .indicesRouting(List.of()) .build(); - Metadata expectedMetadata = Metadata.builder().clusterUUID("cluster-uuid").persistentSettings(Settings.builder().put("readonly", true).build()).build(); + Metadata expectedMetadata = Metadata.builder() + .clusterUUID("cluster-uuid") + .persistentSettings(Settings.builder().put("readonly", true).build()) + .build(); mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expectedMetadata); ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( @@ -3939,6 +4049,55 @@ private void mockBlobContainer( }); } + private void mockBlobContainer( + BlobContainer blobContainer, + ClusterMetadataManifest clusterMetadataManifest, + Map indexMetadataMap, + int codecVersion, + long term, + long version + ) throws IOException { + String manifestFileName = codecVersion >= CODEC_V1 + ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion + : "manifestFileName"; + BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1); + + String manifestPrefix = String.join(DELIMITER, "manifest", RemoteStoreUtils.invertLong(term), RemoteStoreUtils.invertLong(version)) + + DELIMITER; + when(blobContainer.listBlobsByPrefixInSortedOrder(manifestPrefix, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)).thenReturn( + Arrays.asList(blobMetadata) + ); + + BytesReference bytes = RemoteClusterMetadataManifest.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + clusterMetadataManifest, + manifestFileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + when(blobContainer.readBlob(manifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes())); + + clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { + try { + IndexMetadata indexMetadata = indexMetadataMap.get(uploadedIndexMetadata.getIndexUUID()); + if (indexMetadata == null) { + return; + } + String fileName = uploadedIndexMetadata.getUploadedFilename(); + when(blobContainer.readBlob(getFormattedIndexFileName(fileName))).thenAnswer((invocationOnMock) -> { + BytesReference bytesIndexMetadata = INDEX_METADATA_FORMAT.serialize( + indexMetadata, + fileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesIndexMetadata.streamInput().readAllBytes()); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + private void mockBlobContainerForGlobalMetadata( BlobContainer blobContainer, ClusterMetadataManifest clusterMetadataManifest, diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index c439ef209e049..350c6f9ae8f6b 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2413,7 +2413,8 @@ public void onFailure(final Exception e) { clusterService, threadPool, actionFilters, - indexNameExpressionResolver + indexNameExpressionResolver, + null ) ); actions.put( @@ -2455,7 +2456,8 @@ public void onFailure(final Exception e) { clusterService, threadPool, actionFilters, - indexNameExpressionResolver + indexNameExpressionResolver, + null ) ); From 330b2497a9380fa73ee548e61f0c85196eea7acf Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 12 Sep 2024 14:49:45 +0530 Subject: [PATCH 099/357] Add restriction to have a single repository with shallow snapshot v2 setting (#15901) * Add restriction to have a single repository with shallow snapshot v2 setting Signed-off-by: Sachin Kale * Do not allow shallow snapshot v2 repo name to contain SNAPSHOT_PINNED_TIMESTAMP_DELIMITER Signed-off-by: Sachin Kale --------- Signed-off-by: Sachin Kale --- .../RemoteStorePinnedTimestampService.java | 31 ++++- .../repositories/RepositoriesService.java | 79 ++++++++++- .../BlobStoreRepositoryRemoteIndexTests.java | 124 ++++++++++++++++++ 3 files changed, 228 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 3a7734fc0538f..71133615ed056 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -30,6 +30,8 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -75,25 +77,46 @@ public RemoteStorePinnedTimestampService( * and starts the asynchronous update task. */ public void start() { - validateRemoteStoreConfiguration(); + blobContainer = validateAndCreateBlobContainer(settings, repositoriesService.get()); startAsyncUpdateTask(RemoteStoreSettings.getPinnedTimestampsSchedulerInterval()); } - private void validateRemoteStoreConfiguration() { + private static BlobContainer validateAndCreateBlobContainer(Settings settings, RepositoriesService repositoriesService) { final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY ); assert remoteStoreRepo != null : "Remote Segment Store repository is not configured"; - final Repository repository = repositoriesService.get().repository(remoteStoreRepo); + final Repository repository = repositoriesService.repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; - blobContainer = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN)); + return blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(PINNED_TIMESTAMPS_PATH_TOKEN)); } private void startAsyncUpdateTask(TimeValue pinnedTimestampsSchedulerInterval) { asyncUpdatePinnedTimestampTask = new AsyncUpdatePinnedTimestampTask(logger, threadPool, pinnedTimestampsSchedulerInterval, true); } + public static Map> fetchPinnedTimestamps(Settings settings, RepositoriesService repositoriesService) + throws IOException { + BlobContainer blobContainer = validateAndCreateBlobContainer(settings, repositoriesService); + Set pinnedTimestamps = blobContainer.listBlobs().keySet(); + Map> pinningEntityTimestampMap = new HashMap<>(); + for (String pinnedTimestamp : pinnedTimestamps) { + try { + String[] tokens = pinnedTimestamp.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR); + Long timestamp = Long.parseLong(tokens[tokens.length - 1]); + String pinningEntity = pinnedTimestamp.substring(0, pinnedTimestamp.lastIndexOf(PINNED_TIMESTAMPS_FILENAME_SEPARATOR)); + if (pinningEntityTimestampMap.containsKey(pinningEntity) == false) { + pinningEntityTimestampMap.put(pinningEntity, new HashSet<>()); + } + pinningEntityTimestampMap.get(pinningEntity).add(timestamp); + } catch (NumberFormatException e) { + logger.error("Exception while parsing pinned timestamp from {}, skipping this entry", pinnedTimestamp); + } + } + return pinningEntityTimestampMap; + } + /** * Pins a timestamp in the remote store. * diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 68669feb16abc..7da52147661dc 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -68,7 +68,9 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository; +import org.opensearch.snapshots.SnapshotsService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -84,6 +86,7 @@ import java.util.stream.Stream; import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SHALLOW_SNAPSHOT_V2; import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; /** @@ -123,6 +126,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private final RepositoriesStatsArchive repositoriesStatsArchive; private final ClusterManagerTaskThrottler.ThrottlingKey putRepositoryTaskKey; private final ClusterManagerTaskThrottler.ThrottlingKey deleteRepositoryTaskKey; + private final Settings settings; public RepositoriesService( Settings settings, @@ -132,6 +136,7 @@ public RepositoriesService( Map internalTypesRegistry, ThreadPool threadPool ) { + this.settings = settings; this.typesRegistry = typesRegistry; this.internalTypesRegistry = internalTypesRegistry; this.clusterService = clusterService; @@ -173,7 +178,7 @@ public void registerOrUpdateRepository(final PutRepositoryRequest request, final CryptoMetadata.fromRequest(request.cryptoSettings()) ); validate(request.name()); - validateRepositoryMetadataSettings(clusterService, request.name(), request.settings()); + validateRepositoryMetadataSettings(clusterService, request.name(), request.settings(), repositories, settings, this); if (newRepositoryMetadata.cryptoMetadata() != null) { validate(newRepositoryMetadata.cryptoMetadata().keyProviderName()); } @@ -684,7 +689,10 @@ public static void validate(final String identifier) { public static void validateRepositoryMetadataSettings( ClusterService clusterService, final String repositoryName, - final Settings repositoryMetadataSettings + final Settings repositoryMetadataSettings, + Map repositories, + Settings settings, + RepositoriesService repositoriesService ) { // We can add more validations here for repository settings in the future. Version minVersionInCluster = clusterService.state().getNodes().getMinNodeVersion(); @@ -699,6 +707,51 @@ public static void validateRepositoryMetadataSettings( + minVersionInCluster ); } + if (SHALLOW_SNAPSHOT_V2.get(repositoryMetadataSettings)) { + if (minVersionInCluster.onOrAfter(Version.V_2_17_0) == false) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled as some of the nodes in cluster are on version older than " + + Version.V_2_17_0 + + ". Minimum node version in cluster is: " + + minVersionInCluster + ); + } + if (repositoryName.contains(SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled for repository with " + + SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + + " in the name as this delimiter is used to create pinning entity" + ); + } + if (repositoryWithShallowV2Exists(repositories)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled as this setting can be enabled only on one repository " + + " and one or more repositories in the cluster have the setting as enabled" + ); + } + try { + if (pinnedTimestampExistsWithDifferentRepository(repositoryName, settings, repositoriesService)) { + throw new RepositoryException( + repositoryName, + "setting " + + SHALLOW_SNAPSHOT_V2.getKey() + + " cannot be enabled if there are existing snapshots created with shallow V2 " + + "setting using different repository." + ); + } + } catch (IOException e) { + throw new RepositoryException(repositoryName, "Exception while fetching pinned timestamp details"); + } + } // Validation to not allow users to create system repository via put repository call. if (isSystemRepositorySettingPresent(repositoryMetadataSettings)) { throw new RepositoryException( @@ -710,6 +763,28 @@ public static void validateRepositoryMetadataSettings( } } + private static boolean repositoryWithShallowV2Exists(Map repositories) { + return repositories.values().stream().anyMatch(repo -> SHALLOW_SNAPSHOT_V2.get(repo.getMetadata().settings())); + } + + private static boolean pinnedTimestampExistsWithDifferentRepository( + String newRepoName, + Settings settings, + RepositoriesService repositoriesService + ) throws IOException { + Map> pinningEntityTimestampMap = RemoteStorePinnedTimestampService.fetchPinnedTimestamps( + settings, + repositoriesService + ); + for (String pinningEntity : pinningEntityTimestampMap.keySet()) { + String repoNameWithPinnedTimestamps = pinningEntity.split(SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER)[0]; + if (repoNameWithPinnedTimestamps.equals(newRepoName) == false) { + return true; + } + } + return false; + } + private static void ensureRepositoryNotInUse(ClusterState clusterState, String repository) { if (isRepositoryInUse(clusterState, repository)) { throw new IllegalStateException("trying to modify or unregister repository that is currently used"); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 9cca495cced72..e280ab8c7a73c 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -33,6 +33,8 @@ package org.opensearch.repositories.blobstore; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.Settings; @@ -41,13 +43,16 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.index.IndexSettings; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.snapshots.SnapshotsService; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; @@ -64,6 +69,9 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SHALLOW_SNAPSHOT_V2; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** @@ -81,6 +89,7 @@ protected Settings nodeSettings() { .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), tempDir.getParent()) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) .build(); } @@ -373,4 +382,119 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assertThat(snapshotIds, equalTo(originalSnapshots)); } + public void testRepositoryCreationShallowV2() throws Exception { + Client client = client(); + + Settings snapshotRepoSettings1 = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .put(REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(SHALLOW_SNAPSHOT_V2.getKey(), true) + .build(); + + String invalidRepoName = "test" + SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + "repo-1"; + try { + createRepository(client, invalidRepoName, snapshotRepoSettings1); + } catch (RepositoryException e) { + assertEquals( + "[" + + invalidRepoName + + "] setting shallow_snapshot_v2 cannot be enabled for repository with __ in the name as this delimiter is used to create pinning entity", + e.getMessage() + ); + } + + // Create repo with shallow snapshot V2 enabled + createRepository(client, "test-repo-1", snapshotRepoSettings1); + + logger.info("--> verify the repository"); + VerifyRepositoryResponse verifyRepositoryResponse = client.admin().cluster().prepareVerifyRepository("test-repo-1").get(); + assertNotNull(verifyRepositoryResponse.getNodes()); + + GetRepositoriesResponse getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + Settings snapshotRepoSettings2 = Settings.builder() + .put(node().settings()) + .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) + .put(REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(SHALLOW_SNAPSHOT_V2.getKey(), true) + .build(); + + // Create another repo with shallow snapshot V2 enabled, this should fail. + try { + createRepository(client, "test-repo-2", snapshotRepoSettings2); + } catch (RepositoryException e) { + assertEquals( + "[test-repo-2] setting shallow_snapshot_v2 cannot be enabled as this setting can be enabled only on one repository and one or more repositories in the cluster have the setting as enabled", + e.getMessage() + ); + } + + // Disable shallow snapshot V2 setting on test-repo-1 + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + // Create test-repo-2 with shallow snapshot V2 enabled, this should pass now. + createRepository(client, "test-repo-2", snapshotRepoSettings2); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-2").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + final String indexName = "test-idx"; + createIndex(indexName); + ensureGreen(); + indexDocuments(client, indexName); + + // Create pinned timestamp snapshot in test-repo-2 + SnapshotInfo snapshotInfo = createSnapshot("test-repo-2", "test-snap-2", new ArrayList<>()); + assertNotNull(snapshotInfo.snapshotId()); + + // As snapshot is present, even after disabling shallow snapshot setting in test-repo-2, we will not be able to + // enable shallow snapshot v2 setting in test-repo-1 + updateRepository( + client, + "test-repo-2", + Settings.builder().put(snapshotRepoSettings2).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-2").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + try { + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + } catch (RepositoryException e) { + assertEquals( + "[test-repo-1] setting shallow_snapshot_v2 cannot be enabled if there are existing snapshots created with shallow V2 setting using different repository.", + e.getMessage() + ); + } + + // After deleting the snapshot, we will be able to enable shallow snapshot v2 setting in test-repo-1 + AcknowledgedResponse deleteSnapshotResponse = client().admin().cluster().prepareDeleteSnapshot("test-repo-2", "test-snap-2").get(); + + assertAcked(deleteSnapshotResponse); + + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + // Having a snapshot in the same repo should allow disabling and re-enabling shallow snapshot v2 setting + snapshotInfo = createSnapshot("test-repo-1", "test-snap-1", new ArrayList<>()); + assertNotNull(snapshotInfo.snapshotId()); + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put(SHALLOW_SNAPSHOT_V2.getKey(), false).build() + ); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertFalse(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + + updateRepository(client, "test-repo-1", snapshotRepoSettings1); + getRepositoriesResponse = client.admin().cluster().prepareGetRepositories("test-repo-1").get(); + assertTrue(SHALLOW_SNAPSHOT_V2.get(getRepositoriesResponse.repositories().get(0).settings())); + } } From 4223fab40bedd0571324f76f8838ccea4a8b1b8a Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:31:46 -0700 Subject: [PATCH 100/357] Removing task Id from map using remove() (#15918) Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/tasks/TaskCancellationMonitoringService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java index 343d4571593a7..2040703d88c38 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java @@ -125,7 +125,7 @@ public void onTaskCompleted(Task task) { if (!TASKS_TO_TRACK.contains(task.getClass())) { return; } - this.cancelledTaskTracker.entrySet().removeIf(entry -> entry.getKey() == task.getId()); + this.cancelledTaskTracker.remove(task.getId()); } /** From 07029b2513f0cffbd788d95836dbc6cff0b4dbf4 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Fri, 13 Sep 2024 05:53:57 -0700 Subject: [PATCH 101/357] Add the capability to override indices.breaker.total.use_real_memory setting for test clusters (#15906) Signed-off-by: Navneet Verma --- .../org/opensearch/gradle/testclusters/OpenSearchNode.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 268de50340cbf..cd22560af9a96 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -116,7 +116,12 @@ public class OpenSearchNode implements TestClusterConfiguration { private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES; private static final int ADDITIONAL_CONFIG_TIMEOUT = 15; private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS; - private static final List OVERRIDABLE_SETTINGS = Arrays.asList("path.repo", "discovery.seed_providers", "discovery.seed_hosts"); + private static final List OVERRIDABLE_SETTINGS = Arrays.asList( + "path.repo", + "discovery.seed_providers", + "discovery.seed_hosts", + "indices.breaker.total.use_real_memory" + ); private static final int TAIL_LOG_MESSAGES_COUNT = 40; private static final List MESSAGES_WE_DONT_CARE_ABOUT = Arrays.asList( From 260edc5f13debc04858da4e73dfc396eeb459ef9 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 13 Sep 2024 10:44:16 -0400 Subject: [PATCH 102/357] Fix flaky org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15859) Signed-off-by: Andriy Redko --- .../rest/ReactorNetty4StreamingStressIT.java | 56 +++++++------------ 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a978af1b11db4..a853ed56fad32 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,7 +8,6 @@ package org.opensearch.rest; -import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -16,24 +15,20 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; +import java.io.IOException; import java.io.InterruptedIOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import reactor.core.publisher.Flux; -import reactor.test.subscriber.TestSubscriber; +import reactor.test.StepVerifier; +import reactor.test.scheduler.VirtualTimeScheduler; -import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.collection.IsEmptyCollection.empty; public class ReactorNetty4StreamingStressIT extends OpenSearchRestTestCase { @After @@ -49,6 +44,8 @@ public void tearDown() throws Exception { } public void testCloseClientStreamingRequest() throws Exception { + final VirtualTimeScheduler scheduler = VirtualTimeScheduler.create(true); + final AtomicInteger id = new AtomicInteger(0); final Stream stream = Stream.generate( () -> "{ \"index\": { \"_index\": \"test-stress-streaming\", \"_id\": \"" @@ -57,39 +54,28 @@ public void testCloseClientStreamingRequest() throws Exception { + "{ \"name\": \"josh\" }\n" ); + final Duration delay = Duration.ofMillis(1); final StreamingRequest streamingRequest = new StreamingRequest<>( "POST", "/_bulk/stream", - Flux.fromStream(stream).delayElements(Duration.ofMillis(500)).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + Flux.fromStream(stream).delayElements(delay, scheduler).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) ); streamingRequest.addParameter("refresh", "true"); final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); - TestSubscriber subscriber = TestSubscriber.create(); - streamingResponse.getBody().subscribe(subscriber); - - final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - try { - // Await for subscriber to receive at least one chunk - assertBusy(() -> assertThat(subscriber.getReceivedOnNext(), not(empty()))); - - // Close client forceably - executor.schedule(() -> { - client().close(); - return null; - }, 2, TimeUnit.SECONDS); + scheduler.advanceTimeBy(delay); /* emit first element */ - // Await for subscriber to terminate - subscriber.block(Duration.ofSeconds(10)); - assertThat( - subscriber.expectTerminalError(), - anyOf(instanceOf(InterruptedIOException.class), instanceOf(ConnectionClosedException.class)) - ); - } finally { - executor.shutdown(); - if (executor.awaitTermination(1, TimeUnit.SECONDS) == false) { - executor.shutdownNow(); - } - } + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\"")) + .then(() -> { + try { + client().close(); + } catch (final IOException ex) { + throw new UncheckedIOException(ex); + } + }) + .then(() -> scheduler.advanceTimeBy(delay)) + .expectErrorMatches(t -> t instanceof InterruptedIOException) + .verify(); } } From 36c89bf4f975bd2750423d1308d2cd3408d8553f Mon Sep 17 00:00:00 2001 From: Finn Date: Fri, 13 Sep 2024 08:00:37 -0700 Subject: [PATCH 103/357] Update TESTING.md 'Building with extra plugins' documentation (#15893) * Update TESTING.md 'Building with extra plugins' documentation Signed-off-by: Finn Carroll * Typo Signed-off-by: Finn Carroll * Fix TOC Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll --- TESTING.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/TESTING.md b/TESTING.md index de7ab3eefe2f8..9127f005ced75 100644 --- a/TESTING.md +++ b/TESTING.md @@ -39,7 +39,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Expect a specific segment topology](#expect-a-specific-segment-topology) - [Leave environment in an unstable state after test](#leave-environment-in-an-unstable-state-after-test) - [Test coverage analysis](#test-coverage-analysis) -- [Building with extra plugins](#building-with-extra-plugins) +- [Testing with plugins](#testing-with-plugins) - [Environment misc](#environment-misc) # Requirements @@ -552,11 +552,17 @@ Apart from using Gradle, it is also possible to gain insight in code coverage us Please read your IDE documentation for how to attach a debugger to a JVM process. -# Building with extra plugins +# Testing with plugins -Additional plugins may be built alongside OpenSearch, where their dependency on OpenSearch will be substituted with the local OpenSearch build. To add your plugin, create a directory called `opensearch-extra` as a sibling of OpenSearch. Checkout your plugin underneath `opensearch-extra` and the build will automatically pick it up. You can verify the plugin is included as part of the build by checking the projects of the build. +To test a plugin with a custom build of OpenSearch, build OpenSearch and use the `customDistributionUrl` setting supported by each plugin to override the OpenSearch distribution. - ./gradlew projects +For example, in your OpenSearch repository assemble a custom distribution. + + ./gradlew :distribution:archives:linux-tar:assemble + +Then in your plugin repository, substitute in your OpenSearch build + + ./gradlew run -PcustomDistributionUrl="/distribution/archives/linux-tar/build/distributions/opensearch-min-3.0.0-SNAPSHOT-linux-x64.tar.gz" # Environment misc From aaa92aeeb75a745ee7b058cc4b30b734e861cfc4 Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Sat, 14 Sep 2024 04:54:02 +0800 Subject: [PATCH 104/357] Fix case insensitive query on wildcard field (#15882) * fix case insensitive query on wildcard field Signed-off-by: gesong.samuel * fix YAML test Signed-off-by: gesong.samuel * add change log Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Co-authored-by: gesong.samuel --- CHANGELOG.md | 2 +- .../search/270_wildcard_fieldtype_queries.yml | 41 +++++++++++++++++-- .../index/mapper/WildcardFieldMapper.java | 15 +++++-- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3268852cc99f9..ec4ac7504b9c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - +- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index c449a06cf633d..d92538824232d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -56,6 +56,12 @@ setup: id: 6 body: other_field: "test" + - do: + index: + index: test + id: 7 + body: + my_field: "ABCD" - do: indices.refresh: {} @@ -90,8 +96,9 @@ setup: query: term: my_field.lower: "abcd" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -100,8 +107,9 @@ setup: query: term: my_field.lower: "ABCD" - - match: { hits.total.value: 1 } + - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } - do: search: @@ -215,7 +223,7 @@ setup: wildcard: my_field: value: "*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "regexp match-all works": - do: @@ -226,7 +234,7 @@ setup: regexp: my_field: value: ".*" - - match: { hits.total.value: 5 } + - match: { hits.total.value: 6 } --- "terms query on wildcard field matches": - do: @@ -237,3 +245,28 @@ setup: terms: { my_field: ["AbCd"] } - match: { hits.total.value: 1 } - match: { hits.hits.0._id: "5" } +--- +"case insensitive query on wildcard field": + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "5" } + + - do: + search: + index: test + body: + query: + wildcard: + my_field: + value: "AbCd" + case_insensitive: true + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "7" } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 0cb416a9b8370..e43e3bda692e7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; @@ -464,7 +465,7 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo return existsQuery(context); } } else { - approximation = matchAllTermsQuery(name(), requiredNGrams); + approximation = matchAllTermsQuery(name(), requiredNGrams, caseInsensitive); } return new WildcardMatchingQuery(name(), approximation, matchPredicate, value, context, this); } @@ -678,7 +679,7 @@ public Query termsQuery(List values, QueryShardContext context) { StringBuilder pattern = new StringBuilder(); for (Object value : values) { String stringVal = BytesRefs.toString(value); - builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD); + builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal), false), BooleanClause.Occur.SHOULD); expectedValues.add(stringVal); if (pattern.length() > 0) { pattern.append('|'); @@ -688,10 +689,16 @@ public Query termsQuery(List values, QueryShardContext context) { return new WildcardMatchingQuery(name(), builder.build(), expectedValues::contains, pattern.toString(), context, this); } - private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms) { + private static BooleanQuery matchAllTermsQuery(String fieldName, Set terms, boolean caseInsensitive) { BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder(); + Query query; for (String term : terms) { - matchAllTermsBuilder.add(new TermQuery(new Term(fieldName, term)), BooleanClause.Occur.FILTER); + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term)); + } else { + query = new TermQuery(new Term(fieldName, term)); + } + matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER); } return matchAllTermsBuilder.build(); } From bd26056669f41826a500f944869b198149c476a2 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:58:07 +0530 Subject: [PATCH 105/357] Remove TooManyShardsInSnapshotsStatusException (#15877) Signed-off-by: Lakshya Taragi --- .../opensearch/OpenSearchServerException.java | 8 --- ...oManyShardsInSnapshotsStatusException.java | 69 ------------------- .../ExceptionSerializationTests.java | 2 - 3 files changed, 79 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index a1fc61834f69b..b0ab62259d5c0 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -1210,14 +1210,6 @@ public static void registerExceptions() { V_2_17_0 ) ); - registerExceptionHandle( - new OpenSearchExceptionHandle( - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException.class, - org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException::new, - 175, - V_2_17_0 - ) - ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java b/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java deleted file mode 100644 index 1689b3e4941ec..0000000000000 --- a/server/src/main/java/org/opensearch/snapshots/TooManyShardsInSnapshotsStatusException.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.snapshots; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.rest.RestStatus; - -import java.io.IOException; - -/** - * Thrown if the number of shards across the requested resources (snapshot(s) or the index/indices of a particular snapshot) - * breaches the limit of snapshot.max_shards_allowed_in_status_api cluster setting - * - * @opensearch.internal - */ -public class TooManyShardsInSnapshotsStatusException extends SnapshotException { - - public TooManyShardsInSnapshotsStatusException( - final String repositoryName, - final SnapshotId snapshotId, - final String message, - final Throwable cause - ) { - super(repositoryName, snapshotId, message, cause); - } - - public TooManyShardsInSnapshotsStatusException(final String repositoryName, final String message, String... snapshotName) { - super(repositoryName, String.join(", ", snapshotName), message); - } - - public TooManyShardsInSnapshotsStatusException(StreamInput in) throws IOException { - super(in); - } - - @Override - public RestStatus status() { - return RestStatus.REQUEST_ENTITY_TOO_LARGE; - } -} diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index eff312a36dbc6..2e4a2d7bdd59c 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -119,7 +119,6 @@ import org.opensearch.snapshots.SnapshotId; import org.opensearch.snapshots.SnapshotInProgressException; import org.opensearch.snapshots.SnapshotInUseDeletionException; -import org.opensearch.snapshots.TooManyShardsInSnapshotsStatusException; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; import org.opensearch.transport.ActionNotFoundTransportException; @@ -899,7 +898,6 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); - ids.put(175, TooManyShardsInSnapshotsStatusException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); From c709400543a16d7f7d6d644f26bb2f7a10b90caf Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:28:29 +0530 Subject: [PATCH 106/357] Revert "Mute failing snapshot status tests (#15652)" (#15876) This reverts commit 48cf5f061094d89aaa564b97667f4965556a8e49. Signed-off-by: Lakshya Taragi --- .../test/snapshot.status/10_basic.yml | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 819f04407b219..c35f2419bdc91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -25,40 +25,38 @@ setup: snapshot: test_snapshot wait_for_completion: true -# TODO: fix and unmute tests + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_snapshot + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.state: SUCCESS } + - gt: { snapshots.0.stats.incremental.file_count: 0 } + - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } + - gt: { snapshots.0.stats.total.file_count: 0 } + - gt: { snapshots.0.stats.total.size_in_bytes: 0 } + - is_true: snapshots.0.stats.start_time_in_millis +## fast in memory snapshots can take less than one millisecond to complete. + - gte: { snapshots.0.stats.time_in_millis: 0 } + +--- +"Get missing snapshot status throws an exception": + + - do: + catch: /snapshot_missing_exception.+ is missing/ + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + +--- +"Get missing snapshot status succeeds when ignoreUnavailable is true": + + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_nonexistent_snapshot + ignore_unavailable: true -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_snapshot -# -# - is_true: snapshots -# - match: { snapshots.0.snapshot: test_snapshot } -# - match: { snapshots.0.state: SUCCESS } -# - gt: { snapshots.0.stats.incremental.file_count: 0 } -# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } -# - gt: { snapshots.0.stats.total.file_count: 0 } -# - gt: { snapshots.0.stats.total.size_in_bytes: 0 } -# - is_true: snapshots.0.stats.start_time_in_millis -### fast in memory snapshots can take less than one millisecond to complete. -# - gte: { snapshots.0.stats.time_in_millis: 0 } -# -#--- -#"Get missing snapshot status throws an exception": -# -# - do: -# catch: /snapshot_missing_exception.+ is missing/ -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# -#--- -#"Get missing snapshot status succeeds when ignoreUnavailable is true": -# -# - do: -# snapshot.status: -# repository: test_repo_status_1 -# snapshot: test_nonexistent_snapshot -# ignore_unavailable: true -# -# - is_true: snapshots + - is_true: snapshots From 45a8ed41f0c67f8853e36d95c1cc05d921b92739 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:09:17 -0400 Subject: [PATCH 107/357] Bump com.gradle.develocity from 3.18 to 3.18.1 (#15947) Bumps com.gradle.develocity from 3.18 to 3.18.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index b79c2aee135fc..8412d198a2a29 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.develocity" version "3.18" + id "com.gradle.develocity" version "3.18.1" } ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') From e3bbc74868dcba47b4231c8d6e095c2721d019e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:40:39 -0400 Subject: [PATCH 108/357] Bump ch.qos.logback:logback-core from 1.5.6 to 1.5.8 in /test/fixtures/hdfs-fixture (#15946) * Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.6 to 1.5.8. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.6...v_1.5.8) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4ac7504b9c2..cc313fdcdb849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b5cd12ef0c11f..20fd1058a181d 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.6" + api "ch.qos.logback:logback-core:1.5.8" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.26.3' From 81288b1044348d99cce6d23770755a464121f7cc Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 18 Sep 2024 00:28:59 +0530 Subject: [PATCH 109/357] remote publication checksum stats (#15957) * Remote publication checksum stats Signed-off-by: Himshikha Gupta Co-authored-by: Himshikha Gupta --- .../remote/RemoteStatePublicationIT.java | 19 ++++++++++ .../remote/RemoteClusterStateService.java | 6 ++++ .../gateway/remote/RemoteDownloadStats.java | 36 +++++++++++++++++++ .../remote/RemotePersistenceStats.java | 24 ++++++++++--- .../RemoteClusterStateServiceTests.java | 10 ++++++ 5 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index faab3645ae894..ffb9352e8ba47 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -56,6 +56,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteDownloadStats.CHECKSUM_VALIDATION_FAILED_COUNT; import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; @@ -405,10 +406,28 @@ private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(0) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getSuccessCount() > 0); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getFailedCount()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(1).getTotalTimeInMillis() > 0); + assertEquals( + 0, + dataNodeDiscoveryStats.getClusterStateStats() + .getPersistenceStats() + .get(1) + .getExtendedFields() + .get(CHECKSUM_VALIDATION_FAILED_COUNT) + .get() + ); } private Map getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 12d10fd908b44..ece29180f9cf5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1644,6 +1644,12 @@ void validateClusterStateFromChecksum( failedValidation ) ); + if (isFullStateDownload) { + remoteStateStats.stateFullDownloadValidationFailed(); + } else { + remoteStateStats.stateDiffDownloadValidationFailed(); + } + if (isFullStateDownload && remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.FAILURE)) { throw new IllegalStateException( "Cluster state checksums do not match during full state read. Validation failed for " + failedValidation diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java new file mode 100644 index 0000000000000..a8f4b33a19c37 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.coordination.PersistedStateStats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Download stats for remote state + * + * @opensearch.internal + */ +public class RemoteDownloadStats extends PersistedStateStats { + static final String CHECKSUM_VALIDATION_FAILED_COUNT = "checksum_validation_failed_count"; + private AtomicLong checksumValidationFailedCount = new AtomicLong(0); + + public RemoteDownloadStats(String statsName) { + super(statsName); + addToExtendedFields(CHECKSUM_VALIDATION_FAILED_COUNT, checksumValidationFailedCount); + } + + public void checksumValidationFailedCount() { + checksumValidationFailedCount.incrementAndGet(); + } + + public long getChecksumValidationFailedCount() { + return checksumValidationFailedCount.get(); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 417ebdafd3ba7..11f26ac8b3ed9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -18,16 +18,16 @@ public class RemotePersistenceStats { RemoteUploadStats remoteUploadStats; - PersistedStateStats remoteDiffDownloadStats; - PersistedStateStats remoteFullDownloadStats; + RemoteDownloadStats remoteDiffDownloadStats; + RemoteDownloadStats remoteFullDownloadStats; final String FULL_DOWNLOAD_STATS = "remote_full_download"; final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; public RemotePersistenceStats() { remoteUploadStats = new RemoteUploadStats(); - remoteDiffDownloadStats = new PersistedStateStats(DIFF_DOWNLOAD_STATS); - remoteFullDownloadStats = new PersistedStateStats(FULL_DOWNLOAD_STATS); + remoteDiffDownloadStats = new RemoteDownloadStats(DIFF_DOWNLOAD_STATS); + remoteFullDownloadStats = new RemoteDownloadStats(FULL_DOWNLOAD_STATS); } public void cleanUpAttemptFailed() { @@ -90,6 +90,22 @@ public void stateDiffDownloadFailed() { remoteDiffDownloadStats.stateFailed(); } + public void stateDiffDownloadValidationFailed() { + remoteDiffDownloadStats.checksumValidationFailedCount(); + } + + public void stateFullDownloadValidationFailed() { + remoteFullDownloadStats.checksumValidationFailedCount(); + } + + public long getStateDiffDownloadValidationFailed() { + return remoteDiffDownloadStats.getChecksumValidationFailedCount(); + } + + public long getStateFullDownloadValidationFailed() { + return remoteFullDownloadStats.getChecksumValidationFailedCount(); + } + public PersistedStateStats getUploadStats() { return remoteUploadStats; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index b11d5e48bec06..56857285fa8d3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3342,6 +3342,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithNullC anyString(), anyBoolean() ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws IOException { @@ -3374,6 +3375,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(1)).validateClusterStateFromChecksum(manifest, clusterState, ClusterName.DEFAULT.value(), NODE_ID, true); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationModeNone() throws IOException { @@ -3406,6 +3408,7 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw ); mockService.getClusterStateForManifest(ClusterName.DEFAULT.value(), manifest, NODE_ID, true); verify(mockService, times(0)).validateClusterStateFromChecksum(any(), any(), anyString(), anyString(), anyBoolean()); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMismatch() throws IOException { @@ -3448,6 +3451,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException { @@ -3494,6 +3498,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc NODE_ID, true ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksum() throws IOException { @@ -3535,6 +3540,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOException { @@ -3576,6 +3582,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(0, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws IOException { @@ -3616,6 +3623,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws IOException { @@ -3677,6 +3685,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOException { @@ -3738,6 +3747,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio eq(NODE_ID), eq(false) ); + assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 8347d0ec22aa865dea3bc4b2027fb7bdf486fab2 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 18 Sep 2024 03:12:05 +0800 Subject: [PATCH 110/357] Add validation for the search backpressure cancellation settings (#15501) * Fix updating search backpressure settings crashing the cluster Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong * Fix version check Signed-off-by: Gao Binlong * Increase test coverage Signed-off-by: Gao Binlong * Format the code Signed-off-by: Gao Binlong * Optimize some code Signed-off-by: Gao Binlong * Fix typo Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 2 + .../test/cluster.put_settings/10_basic.yml | 113 ++++++++++++++++++ .../opensearch/common/settings/Setting.java | 13 ++ .../SearchBackpressureService.java | 6 +- .../backpressure/SearchBackpressureState.java | 5 +- .../settings/SearchBackpressureSettings.java | 16 ++- .../settings/SearchShardTaskSettings.java | 16 ++- .../settings/SearchTaskSettings.java | 16 ++- .../common/settings/SettingTests.java | 28 +++++ .../SearchBackpressureSettingsTests.java | 28 +++++ 10 files changed, 231 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc313fdcdb849..db41e2eee16f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) +- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) + ### Security [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 107d298b597d3..0f4a45e4591a3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -98,3 +98,116 @@ - match: {error.root_cause.0.type: "illegal_argument_exception"} - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } - match: { status: 400 } + + +--- +"Test setting search backpressure cancellation settings": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_burst: 11 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.1 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.2 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_burst: 12 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.3 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.4 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} + +--- +"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": + - skip: + version: "- 2.99.99" + reason: "Fixed in 3.0.0" + + - do: + catch: /search_backpressure.search_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.0 diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index fea4c165809ba..081029c1c106c 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -1855,6 +1855,10 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting(String key, double defaultValue, Validator validator, Property... properties) { + return new Setting<>(key, Double.toString(defaultValue), Double::parseDouble, validator, properties); + } + /** * A writeable parser for double * @@ -1961,6 +1965,15 @@ public static Setting doubleSetting( ); } + public static Setting doubleSetting( + String key, + Setting fallbackSetting, + Validator validator, + Property... properties + ) { + return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, Double::parseDouble, validator, properties); + } + /// simpleString public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 43b9f8ae87529..e98046ba1dede 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -158,14 +158,16 @@ public SearchBackpressureService( timeNanosSupplier, getSettings().getSearchTaskSettings().getCancellationRateNanos(), getSettings().getSearchTaskSettings().getCancellationBurst(), - getSettings().getSearchTaskSettings().getCancellationRatio() + getSettings().getSearchTaskSettings().getCancellationRatio(), + getSettings().getSearchTaskSettings().getCancellationRate() ), SearchShardTask.class, new SearchBackpressureState( timeNanosSupplier, getSettings().getSearchShardTaskSettings().getCancellationRateNanos(), getSettings().getSearchShardTaskSettings().getCancellationBurst(), - getSettings().getSearchShardTaskSettings().getCancellationRatio() + getSettings().getSearchShardTaskSettings().getCancellationRatio(), + getSettings().getSearchShardTaskSettings().getCancellationRate() ) ); this.settings.getSearchTaskSettings().addListener(searchBackpressureStates.get(SearchTask.class)); diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java index 5f086bd498036..36f5b25e002c3 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureState.java @@ -43,12 +43,15 @@ public class SearchBackpressureState implements CancellationSettingsListener { LongSupplier timeNanosSupplier, double cancellationRateNanos, double cancellationBurst, - double cancellationRatio + double cancellationRatio, + double cancellationRate ) { rateLimiter = new AtomicReference<>(new TokenBucket(timeNanosSupplier, cancellationRateNanos, cancellationBurst)); ratioLimiter = new AtomicReference<>(new TokenBucket(this::getCompletionCount, cancellationRatio, cancellationBurst)); this.timeNanosSupplier = timeNanosSupplier; this.cancellationBurst = cancellationBurst; + this.cancellationRatio = cancellationRatio; + this.cancellationRate = cancellationRate; } public long getCompletionCount() { diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java index 79494eb0d3c24..55a031382f282 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettings.java @@ -61,8 +61,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope @@ -78,7 +84,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.cancellation_rate must be > 0"); + } + }, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java index 6d016c7466362..38213506c55b7 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java @@ -44,8 +44,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_ratio", SearchBackpressureSettings.SETTING_CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -58,7 +64,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_shard_task.cancellation_rate", SearchBackpressureSettings.SETTING_CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_shard_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java index 4b34323b1ddc6..f9af7f9b59fdb 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java @@ -48,8 +48,14 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATIO = Setting.doubleSetting( "search_backpressure.search_task.cancellation_ratio", Defaults.CANCELLATION_RATIO, - 0.0, - 1.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be > 0"); + } + if (value > 1.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_ratio must be <= 1.0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -62,7 +68,11 @@ private static class Defaults { public static final Setting SETTING_CANCELLATION_RATE = Setting.doubleSetting( "search_backpressure.search_task.cancellation_rate", Defaults.CANCELLATION_RATE, - 0.0, + value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("search_backpressure.search_task.cancellation_rate must be > 0"); + } + }, Setting.Property.Dynamic, Setting.Property.NodeScope ); diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index 7ebee680e8e52..c3c399a9d88b2 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -1274,6 +1274,20 @@ public void testFloatParser() throws Exception { public void testDoubleWithDefaultValue() { Setting doubleSetting = Setting.doubleSetting("foo.bar", 42.1); assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(42.1)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", 42.1, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithFallbackValue() { @@ -1282,6 +1296,20 @@ public void testDoubleWithFallbackValue() { assertEquals(doubleSetting.get(Settings.EMPTY), Double.valueOf(2.1)); assertEquals(doubleSetting.get(Settings.builder().put("foo.bar", 3.2).build()), Double.valueOf(3.2)); assertEquals(doubleSetting.get(Settings.builder().put("foo.baz", 3.2).build()), Double.valueOf(3.2)); + + Setting doubleSettingWithValidator = Setting.doubleSetting("foo.bar", fallbackSetting, value -> { + if (value <= 0.0) { + throw new IllegalArgumentException("The setting foo.bar must be >0"); + } + }); + try { + assertThrows( + IllegalArgumentException.class, + () -> doubleSettingWithValidator.get(Settings.builder().put("foo.bar", randomFrom(-1, 0)).build()) + ); + } catch (IllegalArgumentException ex) { + assertEquals("The setting foo.bar must be >0", ex.getMessage()); + } } public void testDoubleWithMinMax() throws Exception { diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java index a02ca3cf877ad..683ada76c7683 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchBackpressureSettingsTests.java @@ -37,4 +37,32 @@ public void testSearchBackpressureSettingValidateInvalidMode() { () -> new SearchBackpressureSettings(settings, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ); } + + public void testInvalidCancellationRate() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_rate", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } + + public void testInvalidCancellationRatio() { + Settings settings1 = Settings.builder().put("search_backpressure.search_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings1, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + + Settings settings2 = Settings.builder().put("search_backpressure.search_shard_task.cancellation_ratio", randomFrom(-1, 0)).build(); + assertThrows( + IllegalArgumentException.class, + () -> new SearchBackpressureSettings(settings2, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + } } From eb5b70370e548cd0f3b51721397df818b3050daa Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 17 Sep 2024 15:44:32 -0400 Subject: [PATCH 111/357] Fix flaky terminaton conditions for org.opensearch.rest.ReactorNetty4StreamingStressIT.testCloseClientStreamingRequest test case (#15959) Signed-off-by: Andriy Redko --- .../org/opensearch/rest/ReactorNetty4StreamingStressIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java index a853ed56fad32..9da456f618ffc 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java @@ -8,6 +8,7 @@ package org.opensearch.rest; +import org.apache.hc.core5.http.ConnectionClosedException; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.StreamingRequest; @@ -75,7 +76,7 @@ public void testCloseClientStreamingRequest() throws Exception { } }) .then(() -> scheduler.advanceTimeBy(delay)) - .expectErrorMatches(t -> t instanceof InterruptedIOException) + .expectErrorMatches(t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException) .verify(); } } From 7c427d9fd588597343654d2f534952f7116d1df4 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:29:12 -0700 Subject: [PATCH 112/357] Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder (#15916) Signed-off-by: David Zane --- CHANGELOG.md | 1 + .../support/ValuesSourceAggregationBuilder.java | 10 +++++++++- .../org/opensearch/search/sort/FieldSortBuilder.java | 8 +++++++- .../bucket/range/RangeAggregationBuilderTests.java | 1 + .../opensearch/search/sort/FieldSortBuilderTests.java | 1 + 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db41e2eee16f5..309d9cb06cc94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) +- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java index 7a73fafb4a809..1ccceb1d77dcb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregationBuilder.java @@ -40,6 +40,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.WithFieldName; import org.opensearch.script.Script; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationInitializationException; @@ -57,7 +58,9 @@ * * @opensearch.internal */ -public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder { +public abstract class ValuesSourceAggregationBuilder> extends AbstractAggregationBuilder + implements + WithFieldName { public static void declareFields( AbstractObjectParser, T> objectParser, @@ -292,6 +295,11 @@ public String field() { return field; } + @Override + public String fieldName() { + return field(); + } + /** * Sets the script to use for this aggregation. */ diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 5cecda1346b90..9825b2cbbe08e 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -65,6 +65,7 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.index.query.WithFieldName; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.SearchSortValuesAndFormats; @@ -86,7 +87,7 @@ * * @opensearch.internal */ -public class FieldSortBuilder extends SortBuilder { +public class FieldSortBuilder extends SortBuilder implements WithFieldName { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldSortBuilder.class); public static final String NAME = "field_sort"; @@ -184,6 +185,11 @@ public String getFieldName() { return this.fieldName; } + @Override + public String fieldName() { + return getFieldName(); + } + /** * Sets the value when a field is missing in a doc. Can also be set to {@code _last} or * {@code _first} to sort missing last or first respectively. diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java index 4362ce48003cc..14532e30f8984 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java @@ -128,6 +128,7 @@ public void testNumericKeys() throws IOException { ); assertThat(builder.getName(), equalTo("test")); assertThat(builder.field(), equalTo("f")); + assertThat(builder.fieldName(), equalTo("f")); assertThat(builder.ranges, equalTo(List.of(new RangeAggregator.Range("1", null, 0d)))); } } diff --git a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java index 9b8cd1b5f1ce0..ced952db555aa 100644 --- a/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/FieldSortBuilderTests.java @@ -196,6 +196,7 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); if (expectedType == SortField.Type.CUSTOM) { assertEquals(builder.getFieldName(), sortField.getField()); + assertEquals(builder.fieldName(), sortField.getField()); } assertEquals(DocValueFormat.RAW, format); } From b2a7136722e9972edcf14ff03abd4f86ae4923b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:58:08 +0800 Subject: [PATCH 113/357] Bump com.microsoft.azure:msal4j from 1.17.0 to 1.17.1 in /plugins/repository-azure (#15945) * Bump com.microsoft.azure:msal4j in /plugins/repository-azure Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.17.0 to 1.17.1. - [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases) - [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt) - [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/commits) --- updated-dependencies: - dependency-name: com.microsoft.azure:msal4j dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 | 1 - plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 309d9cb06cc94..4ed85b630bbd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 5669c5e559ac8..616da0be62c30 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -61,7 +61,7 @@ dependencies { // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" - api 'com.microsoft.azure:msal4j:1.17.0' + api 'com.microsoft.azure:msal4j:1.17.1' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.40' api 'com.nimbusds:content-type:2.3' diff --git a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 deleted file mode 100644 index 34101c989eecd..0000000000000 --- a/plugins/repository-azure/licenses/msal4j-1.17.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7d37157da92b719f250b0023234ac9dda922a2a5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 new file mode 100644 index 0000000000000..46c14e819b630 --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 @@ -0,0 +1 @@ +4eb31a9919d9b103c548af7e37e6f9d9f6e46dbc \ No newline at end of file From 4802d0d744cfd19827ab61c0807f557b768bd1aa Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 18 Sep 2024 07:57:38 -0400 Subject: [PATCH 114/357] Update version in README to 2.18.0 (#15968) Signed-off-by: Craig Perkins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d4a9a671c013..9294d739a0103 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.17.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.0") +[![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) From 3937ccb123aa5ced3c963ec2eef5e07c2085b004 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 15:47:49 -0700 Subject: [PATCH 115/357] [AUTO] [main] Add bwc version 2.17.1. (#15966) * Resolve readme conflicts Signed-off-by: Kunal Kotwani * Fix version identifier Signed-off-by: Kunal Kotwani --------- Signed-off-by: Kunal Kotwani Co-authored-by: Kunal Kotwani --- .ci/bwcVersions | 1 + README.md | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 0f59fec3d824a..ccb47c1a3b724 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -38,4 +38,5 @@ BWC_VERSION: - "2.16.0" - "2.16.1" - "2.17.0" + - "2.17.1" - "2.18.0" diff --git a/README.md b/README.md index 9294d739a0103..03728aee0135c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) [![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") +[![2.17.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.1") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index ca5dd306cf907..b86b8459fb8a8 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -109,6 +109,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 3a1b6d1795f23f2d14c4f2174867ce70ed88525e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:14:00 -0700 Subject: [PATCH 116/357] Refactor branch deletion for backports (#15983) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 387a124b8cb6a..b60c2225db26a 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -7,9 +7,16 @@ on: jobs: delete-branch: runs-on: ubuntu-latest + permissions: + contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions/github-script@v5 + with: + script: | + github.rest.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${context.payload.pull_request.head.ref}`, + }) From 80ff07eb0c5485832e6fcbc46a9d100435ed3051 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 19 Sep 2024 12:50:22 +0530 Subject: [PATCH 117/357] star tree file formats refactoring and fixing offset bug (#15975) Signed-off-by: Bharathwaj G --- .../startree/builder/BaseStarTreeBuilder.java | 31 +++--- .../builder/OffHeapStarTreeBuilder.java | 4 +- .../builder/OnHeapStarTreeBuilder.java | 4 +- .../builder/StarTreeDocsFileManager.java | 43 +++----- .../startree/index/StarTreeValues.java | 57 +++++----- .../utils/SequentialDocValuesIterator.java | 84 ++++++--------- .../SortedNumericStarTreeValuesIterator.java | 32 ++++++ .../iterator/StarTreeValuesIterator.java | 48 +++++++++ .../startree/utils/iterator/package-info.java | 14 +++ .../datacube/startree/StarTreeTestUtils.java | 8 +- .../builder/AbstractStarTreeBuilderTests.java | 101 +++++++++++------- .../SequentialDocValuesIteratorTests.java | 64 +++++------ 12 files changed, 281 insertions(+), 209 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index a1d638616f2aa..5e5814528fcd2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -193,7 +194,9 @@ public List getMetricReaders(SegmentWriteState stat metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); } metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + new SortedNumericStarTreeValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ) ); } metricReaders.add(metricReader); @@ -228,7 +231,7 @@ public void build( dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); } dimensionReaders[i] = new SequentialDocValuesIterator( - fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)) ); } Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); @@ -287,7 +290,7 @@ void appendDocumentsToStarTree(Iterator starTreeDocumentIterat } } - private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException { + private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeDocs) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); StarTreeWriter starTreeWriter = new StarTreeWriter(); @@ -299,7 +302,7 @@ private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDo starTreeField, metricAggregatorInfos, numStarTreeNodes, - numSegmentStarTreeDocument, + numSegmentStarTreeDocuments, numStarTreeDocs, dataFilePointer, totalStarTreeDataLength @@ -400,22 +403,20 @@ protected StarTreeDocument getStarTreeDocument( ) throws IOException { Long[] dims = new Long[numDimensions]; int i = 0; - for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); - Long val = dimensionDocValueIterator.value(currentDocId); + for (SequentialDocValuesIterator dimensionValueIterator : dimensionReaders) { + dimensionValueIterator.nextEntry(currentDocId); + Long val = dimensionValueIterator.value(currentDocId); dims[i] = val; i++; } i = 0; Object[] metrics = new Object[metricReaders.size()]; - for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + for (SequentialDocValuesIterator metricValuesIterator : metricReaders) { + metricValuesIterator.nextEntry(currentDocId); // As part of merge, we traverse the star tree doc values // The type of data stored in metric fields is different from the // actual indexing field they're based on - metrics[i] = metricAggregatorInfos.get(i) - .getValueAggregators() - .toAggregatedValueType(metricDocValuesIterator.value(currentDocId)); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().toAggregatedValueType(metricValuesIterator.value(currentDocId)); i++; } return new StarTreeDocument(dims, metrics); @@ -502,7 +503,7 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { try { - dimensionReaders[i].nextDoc(currentDocId); + dimensionReaders[i].nextEntry(currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -530,7 +531,7 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List mergeStarTrees(List starTreeValuesSub .size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics @@ -164,7 +164,7 @@ Iterator mergeStarTrees(List starTreeValuesSub metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } int currentDocId = 0; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 1a5c906ad413b..13c6d03c4dc3d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -138,7 +138,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -150,7 +150,7 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 779ed77b0540a..15ed153249243 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -55,11 +54,9 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private RandomAccessInput starTreeDocsFileRandomInput; private IndexOutput starTreeDocsFileOutput; private final Map fileToEndDocIdMap; - private final List starTreeDocumentOffsets = new ArrayList<>(); private int currentFileStartDocId; private int numReadableStarTreeDocuments; private int starTreeFileCount = -1; - private int currBytes = 0; private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; @@ -98,7 +95,11 @@ IndexOutput createStarTreeDocumentsFileOutput() throws IOException { public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; int numBytes = writeStarTreeDocument(starTreeDocument, starTreeDocsFileOutput, true); - addStarTreeDocumentOffset(numBytes); + if (docSizeInBytes == -1) { + docSizeInBytes = numBytes; + } else { + assert docSizeInBytes == numBytes; + } numStarTreeDocs++; } @@ -106,7 +107,14 @@ public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isA public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; ensureDocumentReadable(docId); - return readStarTreeDocument(starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId), true); + return readStarTreeDocument(starTreeDocsFileRandomInput, getOffset(docId), true); + } + + /** + * Returns offset for the docId based on the current file start id + */ + private long getOffset(int docId) { + return (long) (docId - currentFileStartDocId) * docSizeInBytes; } @Override @@ -119,19 +127,10 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; - readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId)); + readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId)); return dims; } - private void addStarTreeDocumentOffset(int bytes) { - starTreeDocumentOffsets.add(currBytes); - currBytes += bytes; - if (docSizeInBytes == -1) { - docSizeInBytes = bytes; - } - assert docSizeInBytes == bytes; - } - /** * Load the correct StarTreeDocuments file based on the docId */ @@ -199,7 +198,6 @@ private void loadStarTreeDocumentFile(int docId) throws IOException { * If the operation is only for reading existing documents, a new file is not created. */ private void closeAndMaybeCreateNewFile(boolean shouldCreateFileForAppend, int numStarTreeDocs) throws IOException { - currBytes = 0; if (starTreeDocsFileOutput != null) { fileToEndDocIdMap.put(starTreeDocsFileOutput.getName(), numStarTreeDocs); IOUtils.close(starTreeDocsFileOutput); @@ -232,7 +230,6 @@ private void mergeFiles(int numStarTreeDocs) throws IOException { deleteOldFiles(); fileToEndDocIdMap.clear(); fileToEndDocIdMap.put(mergedOutput.getName(), numStarTreeDocs); - resetStarTreeDocumentOffsets(); } } @@ -259,17 +256,6 @@ private void deleteOldFiles() throws IOException { } } - /** - * Reset the star tree document offsets based on the merged file - */ - private void resetStarTreeDocumentOffsets() { - int curr = 0; - for (int i = 0; i < starTreeDocumentOffsets.size(); i++) { - starTreeDocumentOffsets.set(i, curr); - curr += docSizeInBytes; - } - } - @Override public void close() { try { @@ -288,7 +274,6 @@ public void close() { tmpDirectory.deleteFile(file); } catch (IOException ignored) {} // similar to IOUtils.deleteFilesIgnoringExceptions } - starTreeDocumentOffsets.clear(); fileToEndDocIdMap.clear(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index a34bbbe9ee738..255ad343cde32 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -25,6 +24,8 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; import java.util.ArrayList; @@ -59,14 +60,14 @@ public class StarTreeValues implements CompositeIndexValues { private final StarTreeNode root; /** - * A map containing suppliers for DocIdSetIterators for dimensions. + * A map containing suppliers for StarTreeValues iterators for dimensions. */ - private final Map> dimensionDocValuesIteratorMap; + private final Map> dimensionValuesIteratorMap; /** - * A map containing suppliers for DocIdSetIterators for metrics. + * A map containing suppliers for StarTreeValues iterators for metrics. */ - private final Map> metricDocValuesIteratorMap; + private final Map> metricValuesIteratorMap; /** * A map containing attributes associated with the star tree values. @@ -84,22 +85,22 @@ public class StarTreeValues implements CompositeIndexValues { * * @param starTreeField The StarTreeField object representing the star tree field configuration. * @param root The root node of the star tree. - * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions. - * @param metricDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for metrics. + * @param dimensionValuesIteratorMap A map containing suppliers for StarTreeValues iterators for dimensions. + * @param metricValuesIteratorMap A map containing suppliers for StarTreeValues iterators for metrics. * @param attributes A map containing attributes associated with the star tree values. */ public StarTreeValues( StarTreeField starTreeField, StarTreeNode root, - Map> dimensionDocValuesIteratorMap, - Map> metricDocValuesIteratorMap, + Map> dimensionValuesIteratorMap, + Map> metricValuesIteratorMap, Map attributes, StarTreeMetadata compositeIndexMetadata ) { this.starTreeField = starTreeField; this.root = root; - this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; - this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; + this.dimensionValuesIteratorMap = dimensionValuesIteratorMap; + this.metricValuesIteratorMap = metricValuesIteratorMap; this.attributes = attributes; this.starTreeMetadata = compositeIndexMetadata; } @@ -146,12 +147,12 @@ public StarTreeValues( this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata); // get doc id set iterators for metrics and dimensions - dimensionDocValuesIteratorMap = new LinkedHashMap<>(); - metricDocValuesIteratorMap = new LinkedHashMap<>(); + dimensionValuesIteratorMap = new LinkedHashMap<>(); + metricValuesIteratorMap = new LinkedHashMap<>(); // get doc id set iterators for dimensions for (String dimension : starTreeMetadata.getDimensionFields()) { - dimensionDocValuesIteratorMap.put(dimension, () -> { + dimensionValuesIteratorMap.put(dimension, () -> { try { SortedNumericDocValues dimensionSortedNumericDocValues = null; if (readState != null) { @@ -162,9 +163,9 @@ public StarTreeValues( dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); } } - return getSortedNumericDocValues(dimensionSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); } catch (IOException e) { - throw new RuntimeException("Error loading dimension DocIdSetIterator", e); + throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e); } }); } @@ -177,7 +178,7 @@ public StarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocValuesIteratorMap.put(metricFullName, () -> { + metricValuesIteratorMap.put(metricFullName, () -> { try { SortedNumericDocValues metricSortedNumericDocValues = null; if (readState != null) { @@ -186,7 +187,7 @@ public StarTreeValues( metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo); } } - return getSortedNumericDocValues(metricSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(metricSortedNumericDocValues)); } catch (IOException e) { throw new RuntimeException("Error loading metric DocIdSetIterator", e); } @@ -239,30 +240,30 @@ public Map getAttributes() { } /** - * Returns the DocIdSetIterator for the specified dimension. + * Returns the StarTreeValues iterator for the specified dimension. * * @param dimension The name of the dimension. - * @return The DocIdSetIterator for the specified dimension. + * @return The StarTreeValuesIterator for the specified dimension. */ - public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { + public StarTreeValuesIterator getDimensionValuesIterator(String dimension) { - if (dimensionDocValuesIteratorMap.containsKey(dimension)) { - return dimensionDocValuesIteratorMap.get(dimension).get(); + if (dimensionValuesIteratorMap.containsKey(dimension)) { + return dimensionValuesIteratorMap.get(dimension).get(); } throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); } /** - * Returns the DocIdSetIterator for the specified fully qualified metric name. + * Returns the StarTreeValues iterator for the specified fully qualified metric name. * * @param fullyQualifiedMetricName The fully qualified name of the metric. - * @return The DocIdSetIterator for the specified fully qualified metric name. + * @return The StarTreeValuesIterator for the specified fully qualified metric name. */ - public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) { + public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetricName) { - if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { - return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + if (metricValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { + return metricValuesIteratorMap.get(fullyQualifiedMetricName).get(); } throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 061841d3e140a..9029a451ca4d9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -9,15 +9,19 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; /** - * Coordinates the reading of documents across multiple DocIdSetIterators. - * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. + * Coordinates the reading of documents across multiple StarTreeValuesIterator. + * It encapsulates a single StarTreeValuesIterator and maintains the latest document ID and its associated value. + * + * In case of merge , this will be reading the entries of star tree values and in case of flush this will go through + * the actual segment documents. + * * @opensearch.experimental */ @ExperimentalApi @@ -26,76 +30,56 @@ public class SequentialDocValuesIterator { /** * The doc id set iterator associated for each field. */ - private final DocIdSetIterator docIdSetIterator; + private final StarTreeValuesIterator starTreeValuesIterator; /** - * The value associated with the latest document. + * The id of the latest record/entry. */ - private Long docValue; + private int entryId = -1; - /** - * The id of the latest document. - */ - private int docId = -1; - - /** - * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator. - * - * @param docIdSetIterator the DocIdSetIterator to be associated with this instance - */ - public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { - this.docIdSetIterator = docIdSetIterator; - } - - /** - * Returns the id of the latest document. - * - * @return the id of the latest document - */ - public int getDocId() { - return docId; + public SequentialDocValuesIterator(StarTreeValuesIterator starTreeValuesIterator) { + this.starTreeValuesIterator = starTreeValuesIterator; } /** - * Sets the id of the latest document. + * Returns the ID of the star tree record/entry or the segment document id * - * @param docId the ID of the latest document + * @return the ID of the star tree record/entry or the segment document id */ - private void setDocId(int docId) { - this.docId = docId; + public int getEntryId() { + return entryId; } /** - * Returns the DocIdSetIterator associated with this instance. + * Sets the id of the latest entry. * - * @return the DocIdSetIterator associated with this instance + * @param entryId the ID of the star tree record/entry or the segment document id */ - public DocIdSetIterator getDocIdSetIterator() { - return docIdSetIterator; + private void setEntryId(int entryId) { + this.entryId = entryId; } - public int nextDoc(int currentDocId) throws IOException { + public int nextEntry(int currentEntryId) throws IOException { // if doc id stored is less than or equal to the requested doc id , return the stored doc id - if (docId >= currentDocId) { - return docId; + if (entryId >= currentEntryId) { + return entryId; } - setDocId(this.docIdSetIterator.nextDoc()); - return docId; + setEntryId(this.starTreeValuesIterator.nextEntry()); + return entryId; } - public Long value(int currentDocId) throws IOException { - if (this.getDocIdSetIterator() instanceof SortedNumericDocValues) { - SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) this.getDocIdSetIterator(); - if (currentDocId < 0) { - throw new IllegalStateException("invalid doc id to fetch the next value"); + public Long value(int currentEntryId) throws IOException { + if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) { + if (currentEntryId < 0) { + throw new IllegalStateException("invalid entry id to fetch the next value"); } - if (currentDocId == DocIdSetIterator.NO_MORE_DOCS) { - throw new IllegalStateException("DocValuesIterator is already exhausted"); + if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) { + throw new IllegalStateException("StarTreeValuesIterator is already exhausted"); } - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId != currentDocId) { + if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) { return null; } - return sortedNumericDocValues.nextValue(); + return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue(); } else { throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java new file mode 100644 index 0000000000000..27afdf1479b4e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SortedNumericStarTreeValuesIterator extends StarTreeValuesIterator { + + public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + super(docIdSetIterator); + } + + public long nextValue() throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).nextValue(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java new file mode 100644 index 0000000000000..32866f3e50092 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index in place of DocIdSetIterator to read / traverse the docValues formats. + * This is needed since star tree values are different from segment documents and number of star tree values + * can even exceed segment docs in the worst cases. + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class StarTreeValuesIterator { + + public static final int NO_MORE_ENTRIES = Integer.MAX_VALUE; + protected final DocIdSetIterator docIdSetIterator; + + public StarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + this.docIdSetIterator = docIdSetIterator; + } + + public int entryId() { + return docIdSetIterator.docID(); + } + + public int nextEntry() throws IOException { + return docIdSetIterator.nextDoc(); + } + + public int advance(int target) throws IOException { + return docIdSetIterator.advance(target); + } + + public long cost() { + return docIdSetIterator.cost(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java new file mode 100644 index 0000000000000..3c6444a4a5cac --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This contains classes for StarTreeValues iterators + * + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index b7395b993f67b..7cae1cd25ee93 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -54,7 +54,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } List metricReaders = new ArrayList<>(); @@ -69,7 +69,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } @@ -92,7 +92,7 @@ public static StarTreeDocument getStarTreeDocument( Long[] dims = new Long[dimensionReaders.length]; int i = 0; for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); + dimensionDocValueIterator.nextEntry(currentDocId); Long val = dimensionDocValueIterator.value(currentDocId); dims[i] = val; i++; @@ -100,7 +100,7 @@ public static StarTreeDocument getStarTreeDocument( i = 0; Object[] metrics = new Object[metricReaders.size()]; for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + metricDocValuesIterator.nextEntry(currentDocId); metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i)); i++; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 65adc43ea8bea..b77200f173e71 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -53,6 +53,8 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -395,7 +397,9 @@ SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeD docsWithField.add(i); } } - sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(getSortedNumericMock(dimList, docsWithField)); + sequentialDocValuesIterators[j] = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) + ); } return sequentialDocValuesIterators; } @@ -412,7 +416,9 @@ List getMetricIterators(StarTreeDocument[] starTree docsWithField.add(i); } } - sequentialDocValuesIterators.add(new SequentialDocValuesIterator(getSortedNumericMock(metricslist, docsWithField))); + sequentialDocValuesIterators.add( + new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) + ); } return sequentialDocValuesIterators; } @@ -1985,10 +1991,16 @@ public void testFlushFlow() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); @@ -2081,10 +2093,16 @@ public void testFlushFlowDimsReverse() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( @@ -2508,9 +2526,14 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d1sndv = dimList; SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; - Map> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv) + ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); for (Metric metric : sf.getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -2518,7 +2541,7 @@ private StarTreeValues getStarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocIdSetIterators.put(metricFullName, () -> m1sndv); + metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); } } @@ -3648,18 +3671,18 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3667,7 +3690,7 @@ private StarTreeValues getStarTreeValues( "field2", sf.getMetrics().get(0).getMetrics().get(0).getTypeName() ), - () -> m1sndv + () -> new SortedNumericStarTreeValuesIterator(m1sndv) ); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3675,7 +3698,7 @@ private StarTreeValues getStarTreeValues( "_doc_count", sf.getMetrics().get(1).getMetrics().get(0).getTypeName() ), - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); // metricDocIdSetIterators.put("field2", () -> m1sndv); // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); @@ -4093,24 +4116,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = Map.of( + Map> metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> m1sndv, + () -> new SortedNumericStarTreeValuesIterator(m1sndv), "sf_field2_value_count_metric", - () -> valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), "sf__doc_count_doc_count_metric", - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); StarTreeValues starTreeValues = new StarTreeValues( @@ -4129,24 +4152,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map> f2dimDocIdSetIterators = Map.of( + Map> f2dimDocIdSetIterators = Map.of( "field1", - () -> f2d1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), "field3", - () -> f2d2sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), "field5", - () -> f2d3sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), "field8", - () -> f2d4sndv + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) ); - Map> f2metricDocIdSetIterators = Map.of( + Map> f2metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> f2m1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), "sf_field2_value_count_metric", - () -> f2valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), "sf__doc_count_doc_count_metric", - () -> f2m2sndv + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) ); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java index f56f7d9906ae1..78d63800abd16 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java @@ -9,15 +9,13 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -59,50 +57,32 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getSortedNumeric(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - } - - public void testCreateIterator_UnsupportedType() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - BinaryDocValues iterator = Mockito.mock(BinaryDocValues.class); - when(producer.getBinary(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getBinary(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - when(iterator.nextDoc()).thenReturn(0); - when(iterator.binaryValue()).thenReturn(new BytesRef("123")); + SequentialDocValuesIterator result = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(producer.getSortedNumeric(mockFieldInfo)) + ); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - result.nextDoc(0); - result.value(0); - }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); } public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - sequentialDocValuesIterator.nextDoc(0); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); + sequentialDocValuesIterator.nextEntry(0); long result = sequentialDocValuesIterator.value(0); assertEquals(123L, result); } - public void testGetNextValue_UnsupportedIterator() { - DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { sequentialDocValuesIterator.value(0); }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); - } - - public void testNextDoc() throws IOException { + public void testNextEntry() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); when(iterator.nextDoc()).thenReturn(5); - int result = sequentialDocValuesIterator.nextDoc(5); + int result = sequentialDocValuesIterator.nextEntry(5); assertEquals(5, result); } @@ -110,8 +90,12 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1); - SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2); + SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator1) + ); + SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator2) + ); when(iterator1.nextDoc()).thenReturn(0); when(iterator2.nextDoc()).thenReturn(1); @@ -119,13 +103,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { when(iterator1.nextValue()).thenReturn(9L); when(iterator2.nextValue()).thenReturn(9L); - sequentialDocValuesIterator1.nextDoc(0); - sequentialDocValuesIterator2.nextDoc(0); - assertEquals(0, sequentialDocValuesIterator1.getDocId()); + sequentialDocValuesIterator1.nextEntry(0); + sequentialDocValuesIterator2.nextEntry(0); + assertEquals(0, sequentialDocValuesIterator1.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator1.value(0)); assertNull(sequentialDocValuesIterator2.value(0)); - assertNotEquals(0, sequentialDocValuesIterator2.getDocId()); - assertEquals(1, sequentialDocValuesIterator2.getDocId()); + assertNotEquals(0, sequentialDocValuesIterator2.getEntryId()); + assertEquals(1, sequentialDocValuesIterator2.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator2.value(1)); } } From 6bef1e1d8e2dda8d88c2c8f2a7875b492239a480 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Thu, 19 Sep 2024 16:48:52 +0800 Subject: [PATCH 118/357] Fix infinite loop in nested agg (#15931) Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../search.aggregation/410_nested_aggs.yml | 111 ++++++++++++++++++ .../opensearch/index/mapper/ObjectMapper.java | 13 ++ .../bucket/nested/NestedAggregator.java | 14 +-- .../index/mapper/ObjectMapperTests.java | 47 ++++++++ 5 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed85b630bbd4..636684af939bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml new file mode 100644 index 0000000000000..c115dd4751f8f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/410_nested_aggs.yml @@ -0,0 +1,111 @@ +--- +# The test setup includes: +# - Create nested mapping for test_nested_agg_index index +# - Index two example documents +# - nested agg + +setup: + - do: + indices.create: + index: test_nested_agg_index + body: + mappings: + properties: + a: + type: nested + properties: + b1: + type: keyword + b2: + type: nested + properties: + c: + type: nested + properties: + d: + type: keyword + + - do: + bulk: + refresh: true + body: | + {"index": {"_index": "test_nested_agg_index", "_id": "0"}} + {"a": { "b1": "b11", "b2": { "c": { "d": "d1" } }}} + {"index": {"_index": "test_nested_agg_index", "_id": "1"}} + {"a": { "b1": "b12", "b2": { "c": { "d": "d2" } }}} + +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_nested_agg_index + +--- +"Supported queries": + - skip: + version: " - 2.17.99" + reason: "fixed in 2.18.0" + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 2 } + + # Verify nested aggregation + - do: + search: + body: { + aggs: { + nested_agg: { + nested: { + path: "a" + }, + aggs: { + a_b1: { + terms: { + field: "a.b1" + }, + aggs: { + "c": { + nested: { + path: "a.b2.c" + }, + aggs: { + "d": { + terms: { + field: "a.b2.c.d" + } + } + } + } + } + } + } + } + } + } + + - length: { hits.hits: 2 } + - match: { aggregations.nested_agg.doc_count: 2 } + - length: { aggregations.nested_agg.a_b1.buckets: 2 } + + - match: { aggregations.nested_agg.a_b1.buckets.0.key: "b11" } + - match: { aggregations.nested_agg.a_b1.buckets.0.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.key: "d1" } + - match: { aggregations.nested_agg.a_b1.buckets.0.c.d.buckets.0.doc_count: 1 } + + - match: { aggregations.nested_agg.a_b1.buckets.1.key: "b12" } + - match: { aggregations.nested_agg.a_b1.buckets.1.doc_count: 1 } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.doc_count: 1 } + - length: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets: "1" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.key: "d2" } + - match: { aggregations.nested_agg.a_b1.buckets.1.c.d.buckets.0.doc_count: 1 } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index dd984373fc9df..b93c82d7a5c7c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -171,6 +171,19 @@ public void setIncludeInParent(boolean value) { public void setIncludeInRoot(boolean value) { includeInRoot = new Explicit<>(value, true); } + + public static boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { + if (parentObjectMapper == null || childObjectMapper == null) { + return false; + } + + ObjectMapper parent = childObjectMapper.getParentObjectMapper(mapperService); + while (parent != null && parent != parentObjectMapper) { + childObjectMapper = parent; + parent = childObjectMapper.getParentObjectMapper(mapperService); + } + return parentObjectMapper == parent; + } } /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java index 150efa878f866..db8979d611b4f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -46,7 +46,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lucene.search.Queries; import org.opensearch.core.ParseField; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ObjectMapper; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; @@ -63,6 +62,8 @@ import java.util.List; import java.util.Map; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; + /** * Aggregate all docs that match a nested path * @@ -98,17 +99,6 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2); } - private boolean isParent(ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper, MapperService mapperService) { - if (parentObjectMapper == null) { - return false; - } - ObjectMapper parent; - do { - parent = childObjectMapper.getParentObjectMapper(mapperService); - } while (parent != null && parent != parentObjectMapper); - return parentObjectMapper == parent; - } - @Override public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(ctx); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index cb06bf23d9cbe..b415e1e657f7f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -51,8 +51,12 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; + +import org.mockito.Mockito; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.mapper.ObjectMapper.Nested.isParent; import static org.hamcrest.Matchers.containsString; public class ObjectMapperTests extends OpenSearchSingleNodeTestCase { @@ -568,6 +572,49 @@ public void testCompositeFields() throws Exception { FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } + public void testNestedIsParent() throws Exception { + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("a") + .field("type", "nested") + .startObject("properties") + .field("b1", Collections.singletonMap("type", "keyword")) + .startObject("b2") + .field("type", "nested") + .startObject("properties") + .startObject("c") + .field("type", "nested") + .startObject("properties") + .field("d", Collections.singletonMap("type", "keyword")) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + DocumentMapper documentMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + MapperService mapperService = Mockito.mock(MapperService.class); + Mockito.when(mapperService.getObjectMapper(("a"))).thenReturn(documentMapper.objectMappers().get("a")); + Mockito.when(mapperService.getObjectMapper(("a.b2"))).thenReturn(documentMapper.objectMappers().get("a.b2")); + Mockito.when(mapperService.getObjectMapper(("a.b2.c"))).thenReturn(documentMapper.objectMappers().get("a.b2.c")); + + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a"), documentMapper.objectMappers().get("a.b2"), mapperService)); + assertTrue(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a.b2.c"), mapperService)); + + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2"), documentMapper.objectMappers().get("a"), mapperService)); + assertFalse(isParent(documentMapper.objectMappers().get("a.b2.c"), documentMapper.objectMappers().get("a.b2"), mapperService)); + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); From 620db0a64950cbd6a38642f0be1f592060dac1ee Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:03:02 +0530 Subject: [PATCH 119/357] Explicitly set include discovery nodes flag in reloadSecureSetting request. (#15972) Signed-off-by: Swetha Guptha --- .../cluster/node/reload/NodesReloadSecureSettingsRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index aab7ea54f87c2..d694721feda38 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -60,7 +60,7 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest Date: Thu, 19 Sep 2024 20:48:36 +0800 Subject: [PATCH 120/357] Update version check in yaml test file for bitmap filtering (#15993) Signed-off-by: Gao Binlong --- .../rest-api-spec/test/search/370_bitmap_filtering.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml index d728070adb188..c885e3fbc2446 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml @@ -1,7 +1,7 @@ --- setup: - skip: - version: " - 2.99.99" + version: " - 2.16.99" reason: The bitmap filtering feature is available in 2.17 and later. - do: indices.create: From 77ddfd640cfde672d103c7ff9f25521362cea14e Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:50:47 -0700 Subject: [PATCH 121/357] Fix deleting backport branches workflow (#16003) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index b60c2225db26a..91626a0bc5092 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -11,7 +11,7 @@ jobs: contents: write if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - - name: Delete merged branch + - name: Delete merged branch uses: actions/github-script@v5 with: script: | From 1bc81d3eee07e96b6c6393510b41d194276e204c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 19 Sep 2024 16:55:21 -0400 Subject: [PATCH 122/357] Remove identity-related feature flagged code from the RestController (#15430) * Add authenticate to IdentityPlugin interface Signed-off-by: Craig Perkins * Handle null Signed-off-by: Craig Perkins * Fix tests Signed-off-by: Craig Perkins * Fix ActionModuleTests Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Add DelegatingRestHandlerTests Signed-off-by: Craig Perkins * Address forbiddenApi Signed-off-by: Craig Perkins * Remove authenticate from IdentityPlugin and keep RestController feature flagged code removed Signed-off-by: Craig Perkins * Move RestTokenExtractor to identity-shiro plugin Signed-off-by: Craig Perkins * Remove change in IdentityService Signed-off-by: Craig Perkins * Remove changes in ActionModuleTests Signed-off-by: Craig Perkins * Add tests for RestTokenExtractor Signed-off-by: Craig Perkins * Remove DelegatingRestHandler Signed-off-by: Craig Perkins * Call super instead of keeping a reference to the delegated restHandler Signed-off-by: Craig Perkins * Address code review comments Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../identity/shiro/ShiroIdentityPlugin.java | 47 ++++++++++++++-- .../identity/shiro/ShiroSecurityManager.java | 2 - .../identity/shiro/ShiroSubject.java | 2 - .../identity/shiro/ShiroTokenExtractor.java | 8 +-- .../identity/shiro/ShiroTokenManager.java | 2 - .../shiro/realm/BCryptPasswordMatcher.java | 2 - .../identity/shiro/realm/OpenSearchRealm.java | 6 +-- .../opensearch/identity/shiro/realm/User.java | 2 - .../shiro/ShiroIdentityPluginTests.java | 14 ++--- .../shiro/ShiroTokenExtractorTests.java | 45 ++++++++++++++++ .../org/opensearch/action/ActionModule.java | 2 +- .../org/opensearch/rest/RestController.java | 54 +------------------ .../extensions/ExtensionsManagerTests.java | 3 +- .../opensearch/rest/RestControllerTests.java | 30 ++++------- .../rest/RestHttpResponseHeadersTests.java | 14 +---- .../indices/RestValidateQueryActionTests.java | 12 +---- .../test/rest/RestActionTestCase.java | 15 +----- 18 files changed, 118 insertions(+), 143 deletions(-) rename server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java => plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java (86%) create mode 100644 plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 636684af939bf..b5aef4bb551d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java index af802596ebaa7..2da788242a745 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroIdentityPlugin.java @@ -13,20 +13,28 @@ import org.apache.shiro.SecurityUtils; import org.apache.shiro.mgt.SecurityManager; import org.opensearch.client.Client; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.identity.PluginSubject; import org.opensearch.identity.Subject; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.TokenManager; +import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; @@ -34,14 +42,12 @@ import java.util.Collection; import java.util.Collections; import java.util.function.Supplier; +import java.util.function.UnaryOperator; /** * Identity implementation with Shiro - * - * @opensearch.experimental */ -@ExperimentalApi -public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin { +public final class ShiroIdentityPlugin extends Plugin implements IdentityPlugin, ActionPlugin { private Logger log = LogManager.getLogger(this.getClass()); private final Settings settings; @@ -101,6 +107,37 @@ public TokenManager getTokenManager() { } @Override + public UnaryOperator getRestHandlerWrapper(ThreadContext threadContext) { + return AuthcRestHandler::new; + } + + class AuthcRestHandler extends RestHandler.Wrapper { + + public AuthcRestHandler(RestHandler original) { + super(original); + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + try { + final AuthToken token = ShiroTokenExtractor.extractToken(request); + // If no token was found, continue executing the request + if (token == null) { + // Authentication did not fail so return true. Authorization is handled at the action level. + super.handleRequest(request, channel, client); + return; + } + ShiroSubject shiroSubject = (ShiroSubject) getCurrentSubject(); + shiroSubject.authenticate(token); + // Caller was authorized, forward the request to the handler + super.handleRequest(request, channel, client); + } catch (final Exception e) { + final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, e.getMessage()); + channel.sendResponse(bytesRestResponse); + } + } + } + public PluginSubject getPluginSubject(Plugin plugin) { return new ShiroPluginSubject(threadPool); } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java index 96cf05ac53a1a..0a809dd6c9071 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSecurityManager.java @@ -15,8 +15,6 @@ /** * OpenSearch specific security manager implementation - * - * @opensearch.experimental */ public class ShiroSecurityManager extends DefaultSecurityManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java index 72a168f23c5cd..73ce3f835fc9b 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroSubject.java @@ -17,8 +17,6 @@ /** * Subject backed by Shiro - * - * @opensearch.experimental */ public class ShiroSubject implements UserSubject { private final ShiroTokenManager authTokenHandler; diff --git a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java similarity index 86% rename from server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java rename to plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java index 4bd3ebdded588..86be5ca664daa 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/RestTokenExtractor.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenExtractor.java @@ -5,11 +5,13 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.identity.tokens; +package org.opensearch.identity.shiro; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.core.common.Strings; +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; import org.opensearch.rest.RestRequest; import java.util.Collections; @@ -18,9 +20,9 @@ /** * Extracts tokens from RestRequests used for authentication */ -public class RestTokenExtractor { +public class ShiroTokenExtractor { - private static final Logger logger = LogManager.getLogger(RestTokenExtractor.class); + private static final Logger logger = LogManager.getLogger(ShiroTokenExtractor.class); public final static String AUTH_HEADER_NAME = "Authorization"; diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index a14215aa7655b..cd54bbf9b3124 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -36,8 +36,6 @@ /** * Extracts Shiro's {@link AuthenticationToken} from different types of auth headers - * - * @opensearch.experimental */ class ShiroTokenManager implements TokenManager { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java index a2cb78425929e..f8113101deb70 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/BCryptPasswordMatcher.java @@ -16,8 +16,6 @@ /** * Password matcher for BCrypt - * - * @opensearch.experimental */ public class BCryptPasswordMatcher implements CredentialsMatcher { diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java index ef405a5637ae7..1fc9a1f437a42 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/OpenSearchRealm.java @@ -25,8 +25,6 @@ /** * Internal Realm is a custom realm using the internal OpenSearch IdP - * - * @opensearch.experimental */ public class OpenSearchRealm extends AuthenticatingRealm { private static final String DEFAULT_REALM_NAME = "internal"; @@ -93,7 +91,7 @@ public OpenSearchRealm build() { public User getInternalUser(final String principalIdentifier) throws UnknownAccountException { final User userRecord = internalUsers.get(principalIdentifier); if (userRecord == null) { - throw new UnknownAccountException(); + throw new UnknownAccountException("Incorrect credentials"); } return userRecord; } @@ -131,7 +129,7 @@ protected AuthenticationInfo doGetAuthenticationInfo(final AuthenticationToken t return sai; } else { // Bad password - throw new IncorrectCredentialsException(); + throw new IncorrectCredentialsException("Incorrect credentials"); } } diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java index 35b3348a955d7..1d2d0fed800e2 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/realm/User.java @@ -12,8 +12,6 @@ /** * A non-volatile and immutable object in the storage. - * - * @opensearch.experimental */ public class User { diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java index bc14410d81de0..a15538e48bd66 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroIdentityPluginTests.java @@ -13,7 +13,7 @@ import org.opensearch.identity.IdentityService; import org.opensearch.plugins.IdentityPlugin; import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import java.util.List; @@ -21,27 +21,27 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; public class ShiroIdentityPluginTests extends OpenSearchTestCase { public void testSingleIdentityPluginSucceeds() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList1 = List.of(identityPlugin1); - IdentityService identityService1 = new IdentityService(Settings.EMPTY, threadPool, pluginList1); + IdentityService identityService1 = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList1); assertThat(identityService1.getTokenManager(), is(instanceOf(ShiroTokenManager.class))); - terminate(threadPool); } public void testMultipleIdentityPluginsFail() { - TestThreadPool threadPool = new TestThreadPool(getTestName()); IdentityPlugin identityPlugin1 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin2 = new ShiroIdentityPlugin(Settings.EMPTY); IdentityPlugin identityPlugin3 = new ShiroIdentityPlugin(Settings.EMPTY); List pluginList = List.of(identityPlugin1, identityPlugin2, identityPlugin3); - Exception ex = assertThrows(OpenSearchException.class, () -> new IdentityService(Settings.EMPTY, threadPool, pluginList)); + Exception ex = assertThrows( + OpenSearchException.class, + () -> new IdentityService(Settings.EMPTY, mock(ThreadPool.class), pluginList) + ); assert (ex.getMessage().contains("Multiple identity plugins are not supported,")); - terminate(threadPool); } } diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java new file mode 100644 index 0000000000000..4dc398bacb707 --- /dev/null +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/ShiroTokenExtractorTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.identity.shiro; + +import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.BasicAuthToken; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class ShiroTokenExtractorTests extends OpenSearchTestCase { + + public void testAuthorizationHeaderExtractionWithBasicAuthToken() { + String basicAuthHeader = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(BasicAuthToken.TOKEN_IDENTIFIER + " " + basicAuthHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertThat(extractedToken, instanceOf(BasicAuthToken.class)); + assertThat(extractedToken.asAuthHeaderValue(), equalTo(basicAuthHeader)); + } + + public void testAuthorizationHeaderExtractionWithUnknownToken() { + String authHeader = "foo"; + RestRequest fakeRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( + Map.of(ShiroTokenExtractor.AUTH_HEADER_NAME, List.of(authHeader)) + ).build(); + AuthToken extractedToken = ShiroTokenExtractor.extractToken(fakeRequest); + assertNull(extractedToken); + } +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index fbf90b97d1e8f..3fe0f1dc7cb83 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -579,7 +579,7 @@ public ActionModule( actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()) ); - restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService, identityService); + restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService); } public Map> getActions() { diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 0d6f965c7033f..4f87c01258396 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -41,7 +41,6 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.path.PathTrie; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentType; @@ -56,11 +55,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.http.HttpChunk; import org.opensearch.http.HttpServerTransport; -import org.opensearch.identity.IdentityService; -import org.opensearch.identity.Subject; -import org.opensearch.identity.UserSubject; -import org.opensearch.identity.tokens.AuthToken; -import org.opensearch.identity.tokens.RestTokenExtractor; import org.opensearch.usage.UsageService; import java.io.ByteArrayOutputStream; @@ -125,25 +119,23 @@ public class RestController implements HttpServerTransport.Dispatcher { /** Rest headers that are copied to internal requests made during a rest request. */ private final Set headersToCopy; private final UsageService usageService; - private final IdentityService identityService; public RestController( Set headersToCopy, UnaryOperator handlerWrapper, NodeClient client, CircuitBreakerService circuitBreakerService, - UsageService usageService, - IdentityService identityService + UsageService usageService ) { this.headersToCopy = headersToCopy; this.usageService = usageService; if (handlerWrapper == null) { handlerWrapper = h -> h; // passthrough if no wrapper set } + this.handlerWrapper = handlerWrapper; this.client = client; this.circuitBreakerService = circuitBreakerService; - this.identityService = identityService; registerHandlerNoWrap( RestRequest.Method.GET, "/favicon.ico", @@ -472,11 +464,6 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel return; } } else { - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - if (!handleAuthenticateUser(request, channel)) { - return; - } - } dispatchRequest(request, channel, handler); return; } @@ -587,43 +574,6 @@ private void handleBadRequest(String uri, RestRequest.Method method, RestChannel } } - /** - * Attempts to extract auth token and login. - * - * @return false if there was an error and the request should not continue being dispatched - * */ - private boolean handleAuthenticateUser(final RestRequest request, final RestChannel channel) { - try { - final AuthToken token = RestTokenExtractor.extractToken(request); - // If no token was found, continue executing the request - if (token == null) { - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - final Subject currentSubject = identityService.getCurrentSubject(); - if (currentSubject instanceof UserSubject) { - ((UserSubject) currentSubject).authenticate(token); - logger.debug("Logged in as user " + currentSubject); - } - } catch (final Exception e) { - try { - final BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse( - channel, - RestStatus.UNAUTHORIZED, - e.getMessage() - ); - channel.sendResponse(bytesRestResponse); - } catch (final Exception ex) { - final BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.UNAUTHORIZED, ex.getMessage()); - channel.sendResponse(bytesRestResponse); - } - return false; - } - - // Authentication did not fail so return true. Authorization is handled at the action level. - return true; - } - /** * Get the valid set of HTTP methods for a REST request. */ diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 5ae1bdce48cd5..bf1d52b49cb1f 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -153,8 +153,7 @@ public List> getExtensionSettings() { null, new NodeClient(Settings.EMPTY, threadPool), new NoneCircuitBreakerService(), - new UsageService(), - new IdentityService(Settings.EMPTY, threadPool, List.of()) + new UsageService() ); when(actionModule.getDynamicActionRegistry()).thenReturn(mock(DynamicActionRegistry.class)); when(actionModule.getRestController()).thenReturn(restController); diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index ef9257d746573..f7f1b02847854 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -55,13 +55,11 @@ import org.opensearch.http.HttpResponse; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpStats; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.rest.action.admin.indices.RestCreateIndexAction; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; @@ -97,7 +95,6 @@ public class RestControllerTests extends OpenSearchTestCase { private RestController restController; private HierarchyCircuitBreakerService circuitBreakerService; private UsageService usageService; - private IdentityService identityService; private NodeClient client; @Before @@ -115,11 +112,9 @@ public void setup() { // we can do this here only because we know that we don't adjust breaker settings dynamically in the test inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS); - identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - HttpServerTransport httpServerTransport = new TestHttpServerTransport(); client = new NoOpNodeClient(this.getTestName()); - restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", @@ -140,7 +135,7 @@ public void teardown() throws IOException { } public void testDefaultRestControllerGetAllHandlersContainsFavicon() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); Iterator handlers = restController.getAllHandlers(); assertTrue(handlers.hasNext()); MethodHandlers faviconHandler = handlers.next(); @@ -150,7 +145,7 @@ public void testDefaultRestControllerGetAllHandlersContainsFavicon() { } public void testRestControllerGetAllHandlers() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.PATCH, "/foo", mock(RestHandler.class)); restController.registerHandler(RestRequest.Method.GET, "/foo", mock(RestHandler.class)); @@ -175,7 +170,7 @@ public void testApplyRelevantHeaders() throws Exception { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", true)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("true")); restHeaders.put("header.2", Collections.singletonList("true")); @@ -211,7 +206,7 @@ public void testRequestWithDisallowedMultiValuedHeader() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "bar")); @@ -226,14 +221,7 @@ public void testRequestWithDisallowedMultiValuedHeaderButSameValues() { Set headers = new HashSet<>( Arrays.asList(new RestHeaderDefinition("header.1", true), new RestHeaderDefinition("header.2", false)) ); - final RestController restController = new RestController( - headers, - null, - client, - circuitBreakerService, - usageService, - identityService - ); + final RestController restController = new RestController(headers, null, client, circuitBreakerService, usageService); Map> restHeaders = new HashMap<>(); restHeaders.put("header.1", Collections.singletonList("boo")); restHeaders.put("header.2", Arrays.asList("foo", "foo")); @@ -294,7 +282,7 @@ public void testRegisterWithDeprecatedHandler() { } public void testRegisterSecondMethodWithDifferentNamedWildcard() { - final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService, identityService); + final RestController restController = new RestController(null, null, null, circuitBreakerService, usageService); RestRequest.Method firstMethod = randomFrom(RestRequest.Method.values()); RestRequest.Method secondMethod = randomFrom( @@ -322,7 +310,7 @@ public void testRestHandlerWrapper() throws Exception { final RestController restController = new RestController(Collections.emptySet(), h -> { assertSame(handler, h); return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true); - }, client, circuitBreakerService, usageService, identityService); + }, client, circuitBreakerService, usageService); restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler); RestRequest request = testRestRequest("/wrapped", "{}", MediaTypeRegistry.JSON); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST); @@ -385,7 +373,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() { String content = randomAlphaOfLength((int) Math.round(BREAKER_LIMIT.getBytes() / inFlightRequestsBreaker.getOverhead())); RestRequest request = testRestRequest("/", content, null); AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE); - restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService, identityService); + restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); restController.registerHandler( RestRequest.Method.GET, "/", diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java index 983121a4f481d..b8602cdc20e6a 100644 --- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java +++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java @@ -39,12 +39,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.rest.RestStatus; -import org.opensearch.identity.IdentityService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; import org.opensearch.test.rest.FakeRestRequest; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import java.util.ArrayList; @@ -56,7 +54,6 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; -import static org.mockito.Mockito.mock; public class RestHttpResponseHeadersTests extends OpenSearchTestCase { @@ -106,17 +103,8 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception { new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) ); - final Settings settings = Settings.EMPTY; UsageService usageService = new UsageService(); - final IdentityService identityService = new IdentityService(settings, mock(ThreadPool.class), List.of()); - RestController restController = new RestController( - Collections.emptySet(), - null, - null, - circuitBreakerService, - usageService, - identityService - ); + RestController restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService); // A basic RestHandler handles requests to the endpoint RestHandler restHandler = new RestHandler() { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java index c3cf33f4e9034..6aa1d10d71e50 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -44,7 +44,6 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.search.AbstractSearchTestCase; @@ -61,7 +60,6 @@ import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -75,15 +73,7 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase { private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool); private static UsageService usageService = new UsageService(); - private static IdentityService identityService = new IdentityService(Settings.EMPTY, threadPool, List.of()); - private static RestController controller = new RestController( - emptySet(), - null, - client, - new NoneCircuitBreakerService(), - usageService, - identityService - ); + private static RestController controller = new RestController(emptySet(), null, client, new NoneCircuitBreakerService(), usageService); private static RestValidateQueryAction action = new RestValidateQueryAction(); /** diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java index c7a0fe35b0237..fec1699c9ef64 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java @@ -40,25 +40,20 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; -import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; import org.junit.After; import org.junit.Before; import java.util.Collections; -import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; -import static org.mockito.Mockito.mock; - /** * A common base class for Rest*ActionTests. Provides access to a {@link RestController} * that can be used to register individual REST actions, and test request handling. @@ -70,15 +65,7 @@ public abstract class RestActionTestCase extends OpenSearchTestCase { @Before public void setUpController() { verifyingClient = new VerifyingClient(this.getTestName()); - final IdentityService identityService = new IdentityService(Settings.EMPTY, mock(ThreadPool.class), List.of()); - controller = new RestController( - Collections.emptySet(), - null, - verifyingClient, - new NoneCircuitBreakerService(), - new UsageService(), - identityService - ); + controller = new RestController(Collections.emptySet(), null, verifyingClient, new NoneCircuitBreakerService(), new UsageService()); } @After From 19d8df02444204e2098e012e553066489bb9b679 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 19 Sep 2024 17:22:45 -0400 Subject: [PATCH 123/357] Update version check in yml test file (#15999) Signed-off-by: Gao Binlong Co-authored-by: Gao Binlong --- .../rest-api-spec/test/cluster.put_settings/10_basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 0f4a45e4591a3..ce78709f6ed5e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -103,8 +103,8 @@ --- "Test setting search backpressure cancellation settings": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: cluster.put_settings: @@ -181,8 +181,8 @@ --- "Test setting invalid search backpressure cancellation_rate and cancellation_ratio": - skip: - version: "- 2.99.99" - reason: "Fixed in 3.0.0" + version: "- 2.17.99" + reason: "Fixed in 2.18.0" - do: catch: /search_backpressure.search_task.cancellation_rate must be > 0/ From 94222f1dfaa2fd07598141fb9a894ebca881e201 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 15:08:45 -0700 Subject: [PATCH 124/357] Update big5 workload config to use the ordered documents snapshot (#16005) Signed-off-by: Rishabh Singh --- .github/benchmark-configs.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json index 4ada715d21495..27b7228e1203a 100644 --- a/.github/benchmark-configs.json +++ b/.github/benchmark-configs.json @@ -75,7 +75,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -126,7 +126,7 @@ "SINGLE_NODE_CLUSTER": "true", "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -176,7 +176,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -194,7 +194,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, @@ -212,7 +212,7 @@ "MIN_DISTRIBUTION": "true", "TEST_WORKLOAD": "big5", "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto", - "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", "CAPTURE_NODE_STAT": "true", "TEST_PROCEDURE": "restore-from-snapshot" }, From b3696116965ab45b675731af04e6defc34602f83 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:22:48 -0700 Subject: [PATCH 125/357] Add successfulSearchShardIndices in searchRequestContext before onRequestEnd (#15967) Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../action/search/AbstractSearchAsyncAction.java | 3 +++ .../action/search/SearchRequestContext.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5aef4bb551d3..eebebec672058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index f0fc05c595d6f..836083f91b995 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -754,6 +754,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At } searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits()); searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length); + searchRequestContext.setSuccessfulSearchShardIndices( + results.getSuccessfulResults().map(result -> result.getSearchShardTarget().getIndex()).collect(Collectors.toSet()) + ); onPhaseEnd(searchRequestContext); onRequestEnd(searchRequestContext); listener.onResponse(buildSearchResponse(internalSearchResponse, failures, scrollId, searchContextId)); diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java index 111d9c64550b3..376cf71448d5c 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Supplier; @@ -36,6 +37,7 @@ public class SearchRequestContext { private final Map phaseTookMap; private TotalHits totalHits; private final EnumMap shardStats; + private Set successfulSearchShardIndices; private final SearchRequest searchRequest; private final LinkedBlockingQueue phaseResourceUsage; @@ -141,6 +143,18 @@ public List getPhaseResourceUsage() { public SearchRequest getRequest() { return searchRequest; } + + void setSuccessfulSearchShardIndices(Set successfulSearchShardIndices) { + this.successfulSearchShardIndices = successfulSearchShardIndices; + } + + /** + * @return A {@link List} of {@link String} representing the names of the indices that were + * successfully queried at the shard level. + */ + public Set getSuccessfulSearchShardIndices() { + return successfulSearchShardIndices; + } } enum ShardStatsFieldNames { From 9537d39cc7f4a55c20b7e6f3f3e1a8cec5cb5ec7 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Thu, 19 Sep 2024 18:11:35 -0700 Subject: [PATCH 126/357] add concurrency check to gradle-check workflow (#16007) Signed-off-by: Rishabh Singh --- .github/workflows/gradle-check.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 1b9b30625eb83..1421eeb7f7576 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -8,6 +8,10 @@ on: pull_request_target: types: [opened, synchronize, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: contents: read # to fetch code (actions/checkout) From 2b670cc11f52ada701cbf867ce8ad4527d8b6072 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Fri, 20 Sep 2024 09:36:00 +0530 Subject: [PATCH 127/357] Add support for async deletion in S3BlobContainer (#15621) * Add support for async deletion in S3BlobContainer Signed-off-by: Ashish Singh * Move helper methods to helper class Signed-off-by: Ashish Singh * Minor refactor Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh * Add more tests Signed-off-by: Ashish Singh * Integrate async deletion in the snapshot interactions Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 1 + .../s3/S3BlobStoreRepositoryTests.java | 1 + .../s3/S3RepositoryThirdPartyTests.java | 8 + .../repositories/s3/S3BlobContainer.java | 124 +++++ .../repositories/s3/S3BlobStore.java | 2 +- .../repositories/s3/StatsMetricPublisher.java | 4 + .../s3/async/S3AsyncDeleteHelper.java | 95 ++++ .../s3/S3BlobStoreContainerTests.java | 507 +++++++++++++++++- .../s3/async/S3AsyncDeleteHelperTests.java | 236 ++++++++ .../mocks/MockFsAsyncBlobContainer.java | 12 + .../AsyncMultiStreamBlobContainer.java | 5 + ...syncMultiStreamEncryptedBlobContainer.java | 10 + .../common/settings/ClusterSettings.java | 3 + .../blobstore/BlobStoreRepository.java | 60 ++- .../BlobStoreTransferServiceTests.java | 12 + .../snapshots/BlobStoreFormatTests.java | 12 + 16 files changed, 1086 insertions(+), 6 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index eebebec672058..abcadde83f000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index c5438d58e679d..944de326d144c 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -153,6 +153,7 @@ protected Settings nodeSettings(int nodeOrdinal) { // Disable request throttling because some random values in tests might generate too many failures for the S3 client .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false) .put(S3ClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace("test").getKey(), ProxySettings.ProxyType.DIRECT) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) .put(super.nodeSettings(nodeOrdinal)) .setSecureSettings(secureSettings); diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java index 7db9a0d3ba790..f0e40db965646 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -55,6 +55,14 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put(BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.getKey(), false) + .build(); + } + @Override @Before @SuppressForbidden(reason = "Need to set system property here for AWS SDK v2") diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index b489a3cc85037..1a402e8431e25 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -62,6 +62,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import software.amazon.awssdk.utils.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -90,6 +91,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.repositories.s3.async.S3AsyncDeleteHelper; import org.opensearch.repositories.s3.async.SizeBasedBlockingQ; import org.opensearch.repositories.s3.async.UploadRequest; import org.opensearch.repositories.s3.utils.HttpRangeUtils; @@ -109,6 +111,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE; import static org.opensearch.repositories.s3.S3Repository.MAX_FILE_SIZE_USING_MULTIPART; import static org.opensearch.repositories.s3.S3Repository.MIN_PART_SIZE_USING_MULTIPART; @@ -875,4 +880,123 @@ CompletableFuture getBlobMetadata(S3AsyncClient s3A return SocketAccess.doPrivileged(() -> s3AsyncClient.getObjectAttributes(getObjectAttributesRequest)); } + + @Override + public void deleteAsync(ActionListener completionListener) { + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + ListObjectsV2Request listRequest = ListObjectsV2Request.builder().bucket(blobStore.bucket()).prefix(keyPath).build(); + ListObjectsV2Publisher listPublisher = s3AsyncClient.listObjectsV2Paginator(listRequest); + + AtomicLong deletedBlobs = new AtomicLong(); + AtomicLong deletedBytes = new AtomicLong(); + + CompletableFuture listingFuture = new CompletableFuture<>(); + + listPublisher.subscribe(new Subscriber<>() { + private Subscription subscription; + private final List objectsToDelete = new ArrayList<>(); + private CompletableFuture deletionChain = CompletableFuture.completedFuture(null); + + @Override + public void onSubscribe(Subscription s) { + this.subscription = s; + subscription.request(1); + } + + @Override + public void onNext(ListObjectsV2Response response) { + response.contents().forEach(s3Object -> { + deletedBlobs.incrementAndGet(); + deletedBytes.addAndGet(s3Object.size()); + objectsToDelete.add(s3Object.key()); + }); + + int bulkDeleteSize = blobStore.getBulkDeletesSize(); + if (objectsToDelete.size() >= bulkDeleteSize) { + int fullBatchesCount = objectsToDelete.size() / bulkDeleteSize; + int itemsToDelete = fullBatchesCount * bulkDeleteSize; + + List batchToDelete = new ArrayList<>(objectsToDelete.subList(0, itemsToDelete)); + objectsToDelete.subList(0, itemsToDelete).clear(); + + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + batchToDelete, + deletionChain, + () -> subscription.request(1) + ); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable t) { + listingFuture.completeExceptionally(new IOException("Failed to list objects for deletion", t)); + } + + @Override + public void onComplete() { + if (!objectsToDelete.isEmpty()) { + deletionChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + deletionChain, + null + ); + } + deletionChain.whenComplete((v, throwable) -> { + if (throwable != null) { + listingFuture.completeExceptionally(throwable); + } else { + listingFuture.complete(null); + } + }); + } + }); + + listingFuture.whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure( + throwable instanceof Exception + ? (Exception) throwable + : new IOException("Unexpected error during async deletion", throwable) + ); + } else { + completionListener.onResponse(new DeleteResult(deletedBlobs.get(), deletedBytes.get())); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async deletion", e)); + } + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + if (blobNames.isEmpty()) { + completionListener.onResponse(null); + return; + } + + try (AmazonAsyncS3Reference asyncClientReference = blobStore.asyncClientReference()) { + S3AsyncClient s3AsyncClient = asyncClientReference.get().client(); + + List keysToDelete = blobNames.stream().map(this::buildKey).collect(Collectors.toList()); + + S3AsyncDeleteHelper.executeDeleteChain(s3AsyncClient, blobStore, keysToDelete, CompletableFuture.completedFuture(null), null) + .whenComplete((v, throwable) -> { + if (throwable != null) { + completionListener.onFailure(new IOException("Failed to delete blobs " + blobNames, throwable)); + } else { + completionListener.onResponse(null); + } + }); + } catch (Exception e) { + completionListener.onFailure(new IOException("Failed to initiate async blob deletion", e)); + } + } } diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index f688be9216b8f..90bfa11e18481 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -63,7 +63,7 @@ import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING; import static org.opensearch.repositories.s3.S3Repository.UPLOAD_RETRY_ENABLED; -class S3BlobStore implements BlobStore { +public class S3BlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(S3BlobStore.class); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java index 8d2772d42ebca..9f73c67df3b18 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/StatsMetricPublisher.java @@ -95,6 +95,10 @@ public void publish(MetricCollection metricCollection) { public void close() {} }; + public MetricPublisher getDeleteObjectsMetricPublisher() { + return deleteObjectsMetricPublisher; + } + public MetricPublisher getObjectMetricPublisher = new MetricPublisher() { @Override public void publish(MetricCollection metricCollection) { diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java new file mode 100644 index 0000000000000..eed95c0e68ef1 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelper.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.repositories.s3.S3BlobStore; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +public class S3AsyncDeleteHelper { + private static final Logger logger = LogManager.getLogger(S3AsyncDeleteHelper.class); + + public static CompletableFuture executeDeleteChain( + S3AsyncClient s3AsyncClient, + S3BlobStore blobStore, + List objectsToDelete, + CompletableFuture currentChain, + Runnable afterDeleteAction + ) { + List> batches = createDeleteBatches(objectsToDelete, blobStore.getBulkDeletesSize()); + CompletableFuture newChain = currentChain.thenCompose(v -> executeDeleteBatches(s3AsyncClient, blobStore, batches)); + if (afterDeleteAction != null) { + newChain = newChain.thenRun(afterDeleteAction); + } + return newChain; + } + + static List> createDeleteBatches(List keys, int bulkDeleteSize) { + List> batches = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += bulkDeleteSize) { + batches.add(keys.subList(i, Math.min(keys.size(), i + bulkDeleteSize))); + } + return batches; + } + + static CompletableFuture executeDeleteBatches(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List> batches) { + CompletableFuture allDeletesFuture = CompletableFuture.completedFuture(null); + + for (List batch : batches) { + allDeletesFuture = allDeletesFuture.thenCompose(v -> executeSingleDeleteBatch(s3AsyncClient, blobStore, batch)); + } + + return allDeletesFuture; + } + + static CompletableFuture executeSingleDeleteBatch(S3AsyncClient s3AsyncClient, S3BlobStore blobStore, List batch) { + DeleteObjectsRequest deleteRequest = bulkDelete(blobStore.bucket(), batch, blobStore); + return s3AsyncClient.deleteObjects(deleteRequest).thenApply(S3AsyncDeleteHelper::processDeleteResponse); + } + + static Void processDeleteResponse(DeleteObjectsResponse deleteObjectsResponse) { + if (!deleteObjectsResponse.errors().isEmpty()) { + logger.warn( + () -> new ParameterizedMessage( + "Failed to delete some blobs {}", + deleteObjectsResponse.errors() + .stream() + .map(s3Error -> "[" + s3Error.key() + "][" + s3Error.code() + "][" + s3Error.message() + "]") + .collect(Collectors.toList()) + ) + ); + } + return null; + } + + static DeleteObjectsRequest bulkDelete(String bucket, List blobs, S3BlobStore blobStore) { + return DeleteObjectsRequest.builder() + .bucket(bucket) + .delete( + Delete.builder() + .objects(blobs.stream().map(blob -> ObjectIdentifier.builder().key(blob).build()).collect(Collectors.toList())) + .quiet(true) + .build() + ) + .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().getDeleteObjectsMetricPublisher())) + .build(); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index 654d8a72690c4..2cb11541d924f 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -69,6 +69,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; import org.opensearch.action.LatchedActionListener; import org.opensearch.common.blobstore.BlobContainer; @@ -99,22 +100,28 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; public class S3BlobStoreContainerTests extends OpenSearchTestCase { @@ -1275,6 +1282,504 @@ public void testTransformResponseToInputStreamContainer() throws Exception { assertEquals(inputStream.available(), inputStreamContainer.getInputStream().available()); } + public void testDeleteAsync() throws Exception { + for (int i = 0; i < 100; i++) { + testDeleteAsync(i + 1); + } + } + + private void testDeleteAsync(int bulkDeleteSize) throws InterruptedException { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List s3Objects = new ArrayList<>(); + int numObjects = randomIntBetween(20, 100); + long totalSize = 0; + for (int i = 0; i < numObjects; i++) { + long size = randomIntBetween(1, 100); + s3Objects.add(S3Object.builder().key(randomAlphaOfLength(10)).size(size).build()); + totalSize += size; + } + + final List responseList = new ArrayList<>(); + int size = 0; + while (size < numObjects) { + int toAdd = randomIntBetween(10, 20); + int endIndex = Math.min(numObjects, size + toAdd); + responseList.add(ListObjectsV2Response.builder().contents(s3Objects.subList(size, endIndex)).build()); + size = endIndex; + } + int expectedDeletedObjectsCall = numObjects / bulkDeleteSize + (numObjects % bulkDeleteSize > 0 ? 1 : 0); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + AtomicInteger counter = new AtomicInteger(); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + int currentCounter = counter.getAndIncrement(); + if (currentCounter < responseList.size()) { + subscriber.onNext(responseList.get(currentCounter)); + } + if (currentCounter == responseList.size()) { + subscriber.onComplete(); + } + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference deleteResultRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + deleteResultRef.set(deleteResult); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error("exception during deleteAsync", e); + fail("Unexpected failure: " + e.getMessage()); + } + }); + + latch.await(); + + DeleteResult deleteResult = deleteResultRef.get(); + assertEquals(numObjects, deleteResult.blobsDeleted()); + assertEquals(totalSize, deleteResult.bytesDeleted()); + + verify(s3AsyncClient, times(1)).listObjectsV2Paginator(any(ListObjectsV2Request.class)); + verify(s3AsyncClient, times(expectedDeletedObjectsCall)).deleteObjects(any(DeleteObjectsRequest.class)); + } + + public void testDeleteAsyncFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in listObjectsV2Paginator + RuntimeException simulatedFailure = new RuntimeException("Simulated failure"); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to initiate async deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + + public void testDeleteAsyncListingError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onError(new RuntimeException("Simulated listing error")); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to list objects for deletion", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated listing error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteAsyncDeletionError() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final ListObjectsV2Publisher listPublisher = mock(ListObjectsV2Publisher.class); + doAnswer(invocation -> { + Subscriber subscriber = invocation.getArgument(0); + subscriber.onSubscribe(new Subscription() { + @Override + public void request(long n) { + subscriber.onNext( + ListObjectsV2Response.builder().contents(S3Object.builder().key("test-key").size(100L).build()).build() + ); + subscriber.onComplete(); + } + + @Override + public void cancel() {} + }); + return null; + }).when(listPublisher).subscribe(ArgumentMatchers.>any()); + when(s3AsyncClient.listObjectsV2Paginator(any(ListObjectsV2Request.class))).thenReturn(listPublisher); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete error")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteAsync(new ActionListener<>() { + @Override + public void onResponse(DeleteResult deleteResult) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(CompletionException.class, exceptionRef.get().getClass()); + assertEquals("java.lang.RuntimeException: Simulated delete error", exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("Simulated delete error", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExists() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + int bulkDeleteSize = randomIntBetween(1, 10); + when(blobStore.getBulkDeletesSize()).thenReturn(bulkDeleteSize); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final List blobNames = new ArrayList<>(); + int size = randomIntBetween(10, 100); + for (int i = 0; i < size; i++) { + blobNames.add(randomAlphaOfLength(10)); + } + int expectedDeleteCalls = size / bulkDeleteSize + (size % bulkDeleteSize > 0 ? 1 : 0); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNull(exceptionRef.get()); + + ArgumentCaptor deleteRequestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(expectedDeleteCalls)).deleteObjects(deleteRequestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = deleteRequestCaptor.getAllValues().stream().findAny().get(); + assertEquals(bucketName, capturedRequest.bucket()); + int totalBlobsDeleted = deleteRequestCaptor.getAllValues() + .stream() + .map(r -> r.delete().objects().size()) + .reduce(Integer::sum) + .get(); + assertEquals(blobNames.size(), totalBlobsDeleted); + List deletedKeys = deleteRequestCaptor.getAllValues() + .stream() + .flatMap(r -> r.delete().objects().stream()) + .map(ObjectIdentifier::key) + .collect(Collectors.toList()); + assertTrue(deletedKeys.containsAll(blobNames)); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + // Simulate a failure in deleteObjects + CompletableFuture failedFuture = new CompletableFuture<>(); + failedFuture.completeExceptionally(new RuntimeException("Simulated delete failure")); + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(failedFuture); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull(exceptionRef.get()); + assertEquals(IOException.class, exceptionRef.get().getClass()); + assertEquals("Failed to delete blobs " + blobNames, exceptionRef.get().getMessage()); + assertNotNull(exceptionRef.get().getCause()); + assertEquals("java.lang.RuntimeException: Simulated delete failure", exceptionRef.get().getCause().getMessage()); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsWithEmptyList() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + final S3AsyncClient s3AsyncClient = mock(S3AsyncClient.class); + final AmazonAsyncS3Reference asyncClientReference = mock(AmazonAsyncS3Reference.class); + when(blobStore.asyncClientReference()).thenReturn(asyncClientReference); + AmazonAsyncS3WithCredentials amazonAsyncS3WithCredentials = AmazonAsyncS3WithCredentials.create( + s3AsyncClient, + s3AsyncClient, + s3AsyncClient, + null + ); + when(asyncClientReference.get()).thenReturn(amazonAsyncS3WithCredentials); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List emptyBlobNames = Collections.emptyList(); + + CountDownLatch latch = new CountDownLatch(1); + AtomicBoolean onResponseCalled = new AtomicBoolean(false); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(emptyBlobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + onResponseCalled.set(true); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertTrue("onResponse should have been called", onResponseCalled.get()); + assertNull("No exception should have been thrown", exceptionRef.get()); + + // Verify that no interactions with S3AsyncClient occurred + verifyNoInteractions(s3AsyncClient); + } + + public void testDeleteBlobsAsyncIgnoringIfNotExistsInitializationFailure() throws Exception { + final String bucketName = randomAlphaOfLengthBetween(1, 10); + final BlobPath blobPath = new BlobPath(); + + final S3BlobStore blobStore = mock(S3BlobStore.class); + when(blobStore.bucket()).thenReturn(bucketName); + when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher()); + when(blobStore.getBulkDeletesSize()).thenReturn(1000); + + // Simulate a failure when getting the asyncClientReference + RuntimeException simulatedFailure = new RuntimeException("Simulated initialization failure"); + when(blobStore.asyncClientReference()).thenThrow(simulatedFailure); + + final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + + List blobNames = Arrays.asList("blob1", "blob2", "blob3"); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionRef = new AtomicReference<>(); + + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, new ActionListener() { + @Override + public void onResponse(Void aVoid) { + fail("Expected a failure, but got a success response"); + } + + @Override + public void onFailure(Exception e) { + exceptionRef.set(e); + latch.countDown(); + } + }); + + latch.await(); + + assertNotNull("An exception should have been thrown", exceptionRef.get()); + assertTrue("Exception should be an IOException", exceptionRef.get() instanceof IOException); + assertEquals("Failed to initiate async blob deletion", exceptionRef.get().getMessage()); + assertEquals(simulatedFailure, exceptionRef.get().getCause()); + } + private void mockObjectResponse(S3AsyncClient s3AsyncClient, String bucketName, String blobName, int objectSize) { final InputStream inputStream = new ByteArrayInputStream(randomByteArrayOfLength(objectSize)); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java new file mode 100644 index 0000000000000..d7f924e05cc70 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/S3AsyncDeleteHelperTests.java @@ -0,0 +1,236 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3.async; + +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Error; + +import org.opensearch.repositories.s3.S3BlobStore; +import org.opensearch.repositories.s3.StatsMetricPublisher; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class S3AsyncDeleteHelperTests extends OpenSearchTestCase { + + @Mock + private S3AsyncClient s3AsyncClient; + + @Mock + private S3BlobStore blobStore; + + @Override + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + } + + public void testExecuteDeleteChain() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + null + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + + // Verify that the requests have the expected metric publisher added + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testCreateDeleteBatches() { + List keys = Arrays.asList("key1", "key2", "key3", "key4", "key5", "key6"); + int bulkDeleteSize = 3; + + List> batches = S3AsyncDeleteHelper.createDeleteBatches(keys, bulkDeleteSize); + + assertEquals(2, batches.size()); + assertEquals(Arrays.asList("key1", "key2", "key3"), batches.get(0)); + assertEquals(Arrays.asList("key4", "key5", "key6"), batches.get(1)); + } + + public void testExecuteSingleDeleteBatch() throws Exception { + List batch = Arrays.asList("key1", "key2"); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeSingleDeleteBatch(s3AsyncClient, blobStore, batch); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected request + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient).deleteObjects(requestCaptor.capture()); + + DeleteObjectsRequest capturedRequest = requestCaptor.getValue(); + assertEquals(blobStore.bucket(), capturedRequest.bucket()); + assertEquals(batch.size(), capturedRequest.delete().objects().size()); + assertTrue(capturedRequest.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(batch)); + } + + public void testProcessDeleteResponse() { + DeleteObjectsResponse response = DeleteObjectsResponse.builder() + .errors( + Arrays.asList( + S3Error.builder().key("key1").code("Code1").message("Message1").build(), + S3Error.builder().key("key2").code("Code2").message("Message2").build() + ) + ) + .build(); + + // Call the processDeleteResponse method + S3AsyncDeleteHelper.processDeleteResponse(response); + } + + public void testBulkDelete() { + List blobs = Arrays.asList("key1", "key2", "key3"); + String bucketName = "my-bucket"; + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + DeleteObjectsRequest request = S3AsyncDeleteHelper.bulkDelete(bucketName, blobs, blobStore); + + assertEquals(bucketName, request.bucket()); + assertEquals(blobs.size(), request.delete().objects().size()); + assertTrue(request.delete().objects().stream().map(ObjectIdentifier::key).collect(Collectors.toList()).containsAll(blobs)); + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + + public void testExecuteDeleteBatches() { + List> batches = Arrays.asList(Arrays.asList("key1", "key2"), Arrays.asList("key3", "key4")); + DeleteObjectsResponse expectedResponse = DeleteObjectsResponse.builder().build(); + + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(CompletableFuture.completedFuture(expectedResponse)); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture future = S3AsyncDeleteHelper.executeDeleteBatches(s3AsyncClient, blobStore, batches); + + assertNotNull(future); + assertTrue(future.isDone()); + assertFalse(future.isCompletedExceptionally()); + future.join(); // Wait for the CompletableFuture to complete + + // Verify that the deleteObjects method of S3AsyncClient was called with the expected requests + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3AsyncClient, times(2)).deleteObjects(requestCaptor.capture()); + + List capturedRequests = requestCaptor.getAllValues(); + assertEquals(2, capturedRequests.size()); + for (DeleteObjectsRequest request : capturedRequests) { + assertNotNull(request.overrideConfiguration()); + assertTrue(request.overrideConfiguration().get().metricPublishers().contains(mockDeleteObjectsMetricPublisher)); + } + } + + public void testExecuteDeleteChainWithAfterDeleteAction() { + List objectsToDelete = Arrays.asList("key1", "key2", "key3"); + CompletableFuture currentChain = CompletableFuture.completedFuture(null); + Runnable afterDeleteAction = mock(Runnable.class); + + // Mock the deleteObjects method of S3AsyncClient + when(s3AsyncClient.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn( + CompletableFuture.completedFuture(DeleteObjectsResponse.builder().build()) + ); + + // Mock the getBulkDeletesSize method of S3BlobStore + when(blobStore.getBulkDeletesSize()).thenReturn(2); + + // Mock the getStatsMetricPublisher method of S3BlobStore to return a non-null value + StatsMetricPublisher mockMetricPublisher = mock(StatsMetricPublisher.class); + MetricPublisher mockDeleteObjectsMetricPublisher = mock(MetricPublisher.class); + when(blobStore.getStatsMetricPublisher()).thenReturn(mockMetricPublisher); + when(mockMetricPublisher.getDeleteObjectsMetricPublisher()).thenReturn(mockDeleteObjectsMetricPublisher); + + CompletableFuture newChain = S3AsyncDeleteHelper.executeDeleteChain( + s3AsyncClient, + blobStore, + objectsToDelete, + currentChain, + afterDeleteAction + ); + + // Verify that the newChain is completed without any exceptions + assertNotNull(newChain); + assertTrue(newChain.isDone()); + assertFalse(newChain.isCompletedExceptionally()); + + // Verify that the afterDeleteAction was called + verify(afterDeleteAction).run(); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java index d45b4e3deb798..875f11203281a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java @@ -1,3 +1,4 @@ + /* * SPDX-License-Identifier: Apache-2.0 * @@ -12,6 +13,7 @@ import org.opensearch.common.StreamContext; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -146,4 +148,14 @@ public boolean remoteIntegrityCheckSupported() { private boolean isSegmentFile(String filename) { return !filename.endsWith(".tlog") && !filename.endsWith(".ckp"); } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new UnsupportedOperationException("deleteAsync"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new UnsupportedOperationException("deleteBlobsAsyncIgnoringIfNotExists"); + } } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java index 97f304d776f5c..b769cdc2fe7ab 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamBlobContainer.java @@ -14,6 +14,7 @@ import org.opensearch.core.action.ActionListener; import java.io.IOException; +import java.util.List; /** * An extension of {@link BlobContainer} that adds {@link AsyncMultiStreamBlobContainer#asyncBlobUpload} to allow @@ -48,4 +49,8 @@ public interface AsyncMultiStreamBlobContainer extends BlobContainer { * by underlying blobContainer. In this case, caller doesn't need to ensure integrity of data. */ boolean remoteIntegrityCheckSupported(); + + void deleteAsync(ActionListener completionListener); + + void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener); } diff --git a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java index 82bc7a0baed50..286c01f9dca44 100644 --- a/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java +++ b/server/src/main/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainer.java @@ -171,4 +171,14 @@ private InputStreamContainer decryptInputStreamContainer(InputStreamContainer in return new InputStreamContainer(decryptedStream, adjustedLength, adjustedPos); } } + + @Override + public void deleteAsync(ActionListener completionListener) { + blobContainer.deleteAsync(completionListener); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + blobContainer.deleteBlobsAsyncIgnoringIfNotExists(blobNames, completionListener); + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 09832e2b41b6d..ecdd23530c648 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -779,7 +779,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED, RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX, + + // Snapshot related Settings BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, + BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index b954560c1bc94..0292cecc36a81 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -50,6 +50,7 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.StepListener; import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.RepositoryCleanupInProgress; @@ -69,6 +70,7 @@ import org.opensearch.common.Randomness; import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; @@ -180,6 +182,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -353,6 +356,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.Final ); + /** + * Controls the fixed prefix for the snapshot shard blob path. cluster.snapshot.async-deletion.enable + */ + public static final Setting SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING = Setting.boolSetting( + "cluster.snapshot.async-deletion.enable", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + protected volatile boolean supportURLRepo; private volatile int maxShardBlobDeleteBatch; @@ -446,6 +459,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final String snapshotShardPathPrefix; + private volatile boolean enableAsyncDeletion; + /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart @@ -498,6 +513,8 @@ protected BlobStoreRepository( this.recoverySettings = recoverySettings; this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings()); this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings()); + this.enableAsyncDeletion = SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, this::setEnableAsyncDeletion); } @Override @@ -2082,7 +2099,7 @@ private void executeOneStaleIndexDelete( } // Finally, we delete the [base_path]/indexId folder - deleteResult = deleteResult.add(indexEntry.getValue().delete()); // Deleting the index folder + deleteResult = deleteResult.add(deleteContainer(indexEntry.getValue())); // Deleting the index folder logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); return deleteResult; } catch (IOException e) { @@ -2115,6 +2132,21 @@ private void executeOneStaleIndexDelete( })); } + private DeleteResult deleteContainer(BlobContainer container) throws IOException { + long startTime = System.nanoTime(); + DeleteResult deleteResult; + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + // Use deleteAsync and wait for the result + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteAsync(future); + deleteResult = future.actionGet(); + } else { + deleteResult = container.delete(); + } + logger.debug(new ParameterizedMessage("[{}] Deleted {} in {}ns", metadata.name(), container.path(), startTime - System.nanoTime())); + return deleteResult; + } + /** * Cleans up the remote store directory if needed. *

This method cleans up segments in the remote store directory for deleted indices. @@ -2318,7 +2350,7 @@ void releaseRemoteStoreLocksAndCleanup( * @return A DeleteResult object representing the result of the deletion operation. * @throws IOException If an I/O error occurs during the deletion process. */ - private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { + private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException, ExecutionException, InterruptedException { // If the provided ShardInfo is null, return a zero DeleteResult if (shardInfo == null) { return DeleteResult.ZERO; @@ -2330,7 +2362,7 @@ private DeleteResult deleteShardData(ShardInfo shardInfo) throws IOException { // Iterate over the shards and delete each shard's data for (int i = 0; i < shardInfo.getShardCount(); i++) { // Call the delete method on the shardContainer and accumulate the result - deleteResult = deleteResult.add(shardContainer(shardInfo.getIndexId(), i).delete()); + deleteResult = deleteResult.add(deleteContainer(shardContainer(shardInfo.getIndexId(), i))); } // Return the accumulated DeleteResult @@ -2714,7 +2746,23 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna private void deleteFromContainer(BlobContainer container, List blobs) throws IOException { logger.trace(() -> new ParameterizedMessage("[{}] Deleting {} from [{}]", metadata.name(), blobs, container.path())); - container.deleteBlobsIgnoringIfNotExists(blobs); + long startTime = System.nanoTime(); + if (enableAsyncDeletion && container instanceof AsyncMultiStreamBlobContainer) { + PlainActionFuture future = new PlainActionFuture<>(); + ((AsyncMultiStreamBlobContainer) container).deleteBlobsAsyncIgnoringIfNotExists(blobs, future); + future.actionGet(); + } else { + container.deleteBlobsIgnoringIfNotExists(blobs); + } + logger.debug( + () -> new ParameterizedMessage( + "[{}] Deletion {} from [{}] took {}ns", + metadata.name(), + blobs, + container.path(), + System.nanoTime() - startTime + ) + ); } private BlobPath indicesPath() { @@ -4565,4 +4613,8 @@ public String toString() { return name; } } + + public void setEnableAsyncDeletion(boolean enableAsyncDeletion) { + this.enableAsyncDeletion = enableAsyncDeletion; + } } diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java index 10e4cc6cfb1ef..fddc6c0c94005 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -51,6 +52,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -340,5 +342,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } } } diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index 95a8267734a07..72798f7691eb1 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.blobstore.fs.FsBlobContainer; import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.blobstore.stream.read.ReadContext; @@ -63,6 +64,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -335,5 +337,15 @@ public boolean remoteIntegrityCheckSupported() { public BlobContainer getDelegate() { return delegate; } + + @Override + public void deleteAsync(ActionListener completionListener) { + throw new RuntimeException("deleteAsync not supported"); + } + + @Override + public void deleteBlobsAsyncIgnoringIfNotExists(List blobNames, ActionListener completionListener) { + throw new RuntimeException("deleteBlobsAsyncIgnoringIfNotExists not supported"); + } } } From f0ea056a93383c4ca7cac361fdf8c73e8afc3941 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 20 Sep 2024 08:42:40 -0400 Subject: [PATCH 128/357] Update protobuf from 3.25.4 to 3.25.5 (#16011) * Update protobuf from 3.25.4 to 3.25.5 Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- server/licenses/protobuf-java-3.25.4.jar.sha1 | 1 - server/licenses/protobuf-java-3.25.5.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 server/licenses/protobuf-java-3.25.4.jar.sha1 create mode 100644 server/licenses/protobuf-java-3.25.5.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index abcadde83f000..32fdfab619b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) +- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) ### Changed diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 0c6d0536dd1d5..63817289e80c0 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -20,7 +20,7 @@ woodstox = 6.4.0 kotlin = 1.7.10 antlr4 = 4.13.1 guava = 32.1.1-jre -protobuf = 3.25.4 +protobuf = 3.25.5 jakarta_annotation = 1.3.5 google_http_client = 1.44.1 tdigest = 3.3 diff --git a/server/licenses/protobuf-java-3.25.4.jar.sha1 b/server/licenses/protobuf-java-3.25.4.jar.sha1 deleted file mode 100644 index 21b6fdfd24dc8..0000000000000 --- a/server/licenses/protobuf-java-3.25.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -43fcb86e4a411516c7fc681450f1516de0b862a2 \ No newline at end of file diff --git a/server/licenses/protobuf-java-3.25.5.jar.sha1 b/server/licenses/protobuf-java-3.25.5.jar.sha1 new file mode 100644 index 0000000000000..72b42c9efc85a --- /dev/null +++ b/server/licenses/protobuf-java-3.25.5.jar.sha1 @@ -0,0 +1 @@ +5ae5c9ec39930ae9b5a61b32b93288818ec05ec1 \ No newline at end of file From 4963792c175ab3eb30a493e81360dce95b5b2811 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 20 Sep 2024 22:24:20 +0800 Subject: [PATCH 129/357] Add more yaml rest tests for search backpressure settings (#16012) Signed-off-by: Gao Binlong --- .../test/cluster.put_settings/10_basic.yml | 142 -------- .../test/search.backpressure/10_basic.yml | 340 ++++++++++++++++++ 2 files changed, 340 insertions(+), 142 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index ce78709f6ed5e..825bac9f91649 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -69,145 +69,3 @@ include_defaults: true - match: {defaults.node.attr.testattr: "test"} - ---- -"Test set search backpressure mode": - - - do: - cluster.put_settings: - body: - persistent: - search_backpressure.mode: "monitor_only" - - - match: {persistent: {search_backpressure: {mode: "monitor_only"}}} - ---- -"Test set invalid search backpressure mode": - - - skip: - version: "- 2.8.99" - reason: "Fixed in 2.9.0" - - - do: - catch: bad_request - cluster.put_settings: - body: - persistent: - search_backpressure.mode: "monitor-only" - - - match: {error.root_cause.0.type: "illegal_argument_exception"} - - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } - - match: { status: 400 } - - ---- -"Test setting search backpressure cancellation settings": - - skip: - version: "- 2.17.99" - reason: "Fixed in 2.18.0" - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_burst: 11 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_rate: 0.1 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_ratio: 0.2 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_burst: 12 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_rate: 0.3 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} - - - do: - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_ratio: 0.4 - - is_true: acknowledged - - - do: - cluster.get_settings: - flat_settings: false - - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} - ---- -"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": - - skip: - version: "- 2.17.99" - reason: "Fixed in 2.18.0" - - - do: - catch: /search_backpressure.search_task.cancellation_rate must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_rate: 0.0 - - - do: - catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_task.cancellation_ratio: 0.0 - - - do: - catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_rate: 0.0 - - - do: - catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ - cluster.put_settings: - body: - transient: - search_backpressure.search_shard_task.cancellation_ratio: 0.0 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml new file mode 100644 index 0000000000000..2862dcce072c4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.backpressure/10_basic.yml @@ -0,0 +1,340 @@ +--- +"Test set search backpressure mode": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + - do: + cluster.put_settings: + body: + persistent: + search_backpressure.mode: "monitor_only" + + - match: {persistent: {search_backpressure: {mode: "monitor_only"}}} + +--- +"Test set invalid search backpressure mode": + - skip: + version: "- 2.7.99" + reason: "Fixed in 2.8.0" + + - do: + catch: bad_request + cluster.put_settings: + body: + persistent: + search_backpressure.mode: "monitor-only" + + - match: {error.root_cause.0.type: "illegal_argument_exception"} + - match: { error.root_cause.0.reason: "Invalid SearchBackpressureMode: monitor-only" } + - match: { status: 400 } + +--- +"Test setting search backpressure cancellation settings": + - skip: + version: "- 2.17.99" + reason: "Fixed in 2.18.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_burst: 11 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_burst: "11"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.1 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_rate: "0.1"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.2 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cancellation_ratio: "0.2"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_burst: 12 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_burst: "12"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.3 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_rate: "0.3"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.4 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cancellation_ratio: "0.4"} + +--- +"Test setting invalid search backpressure cancellation_rate and cancellation_ratio": + - skip: + version: "- 2.17.99" + reason: "Fixed in 2.18.0" + + - do: + catch: /search_backpressure.search_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cancellation_ratio: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_rate must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_rate: 0.0 + + - do: + catch: /search_backpressure.search_shard_task.cancellation_ratio must be > 0/ + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cancellation_ratio: 0.0 + +--- +"Test setting search backpressure node settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.num_successive_breaches: 10 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.num_successive_breaches: "10"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.cpu_threshold: 0.85 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.cpu_threshold: "0.85"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.node_duress.heap_threshold: 0.75 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.node_duress.heap_threshold: "0.75"} + + +--- +"Test setting search backpressure search_task settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.elapsed_time_millis_threshold: 50000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.elapsed_time_millis_threshold: "50000"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_percent_threshold: 0.01 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_percent_threshold: "0.01"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.total_heap_percent_threshold: 0.08 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.total_heap_percent_threshold: "0.08"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_variance: 3.0 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_variance: "3.0"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.heap_moving_average_window_size: 50 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.heap_moving_average_window_size: "50"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_task.cpu_time_millis_threshold: 40000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_task.cpu_time_millis_threshold: "40000"} + +--- +"Test setting search backpressure search_shard_task settings": + - skip: + version: "- 2.3.99" + reason: "Introduced in 2.4.0" + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.elapsed_time_millis_threshold: 50000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.elapsed_time_millis_threshold: "50000"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_percent_threshold: 0.01 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_percent_threshold: "0.01"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.total_heap_percent_threshold: 0.08 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.total_heap_percent_threshold: "0.08"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_variance: 3.0 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_variance: "3.0"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.heap_moving_average_window_size: 50 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.heap_moving_average_window_size: "50"} + + - do: + cluster.put_settings: + body: + transient: + search_backpressure.search_shard_task.cpu_time_millis_threshold: 40000 + - is_true: acknowledged + + - do: + cluster.get_settings: + flat_settings: false + - match: {transient.search_backpressure.search_shard_task.cpu_time_millis_threshold: "40000"} From 05dab3b7eb54a361af3583a322f0a748d6412836 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Fri, 20 Sep 2024 15:21:39 -0700 Subject: [PATCH 130/357] Avoid infinite loop in flat_object parsing (#15985) * Avoid infinite loop in flat_object parsing We had logic in flat_object parsing that would: 1. Try parsing a flat object field that is not an object or null. 2. Would see an END_ARRAY token, ignore it, and not advance the parser. Combined, this would create a scenario where passing an array of strings for a flat_object would parse the string values, then loop infinitely on the END_ARRAY token. Signed-off-by: Michael Froh * Remove some unused code and add more tests The removed code does not actually seem to affect the logic. Also, I want to be 100% sure that every call to parseToken is guaranteed to call parser.nextToken() at some point. Signed-off-by: Michael Froh * Remove unused parameter from parseToken Thanks for the reminder, @kkewwei! Signed-off-by: Michael Froh * Add skip for newly-added test The test fails on MixedClusterClientYamlTestSuiteIT because 2.x still has the infinite loop until backport. Signed-off-by: Michael Froh --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../test/index/90_flat_object.yml | 48 +++++++++++++++- .../xcontent/JsonToStringXContentParser.java | 37 +++--------- .../index/mapper/FlatObjectFieldMapper.java | 57 +++++++++++-------- 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fdfab619b3e..ae0b0e0e7a235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index 83d3d273ebd93..c0fc0090abedf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -62,7 +62,6 @@ setup: }, "required_matches": 1 } - # Do index refresh - do: indices.refresh: @@ -74,7 +73,52 @@ teardown: - do: indices.delete: index: test - +--- +"Invalid docs": + - skip: + version: "- 2.99.99" + reason: "parsing of these objects would infinite loop prior to 2.18" + # The following documents are invalid. + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": [ "Arrays in Action" ], + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": "Strings in Action", + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": 12345, + "required_matches": 1 + } + - do: + catch: /parsing_exception/ + index: + index: test + id: 3 + body: { + "ISBN13": "V12154942123242", + "catalog": [ 12345 ], + "required_matches": 1 + } --- # Verify that mappings under the catalog field did not expand # and no dynamic fields were created. diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 2f60fc8f69f87..95a8d9c9495f2 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -9,6 +9,7 @@ package org.opensearch.common.xcontent; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.AbstractXContentParser; @@ -73,7 +74,7 @@ public XContentParser parseObject() throws IOException { builder.startObject(); LinkedList path = new LinkedList<>(Collections.singleton(fieldTypeName)); while (currentToken() != Token.END_OBJECT) { - parseToken(path, null); + parseToken(path); } // deduplication the fieldName,valueList,valueAndPathList builder.field(this.fieldTypeName, new HashSet<>(keyList)); @@ -87,14 +88,11 @@ public XContentParser parseObject() throws IOException { /** * @return true if the child object contains no_null value, false otherwise */ - private boolean parseToken(Deque path, String currentFieldName) throws IOException { - if (path.size() == 1 && processNoNestedValue()) { - return true; - } + private boolean parseToken(Deque path) throws IOException { boolean isChildrenValueValid = false; boolean visitFieldName = false; if (this.parser.currentToken() == Token.FIELD_NAME) { - currentFieldName = this.parser.currentName(); + final String currentFieldName = this.parser.currentName(); path.addLast(currentFieldName); // Pushing onto the stack *must* be matched by pop visitFieldName = true; String parts = currentFieldName; @@ -106,23 +104,21 @@ private boolean parseToken(Deque path, String currentFieldName) throws I } this.keyList.add(parts); // parts has no dot, so either it's the original fieldName or it's the last part this.parser.nextToken(); // advance to the value of fieldName - isChildrenValueValid = parseToken(path, currentFieldName); // parse the value for fieldName (which will be an array, an object, - // or a primitive value) + isChildrenValueValid = parseToken(path); // parse the value for fieldName (which will be an array, an object, + // or a primitive value) path.removeLast(); // Here is where we pop fieldName from the stack (since we're done with the value of fieldName) // Note that whichever other branch we just passed through has already ended with nextToken(), so we // don't need to call it. } else if (this.parser.currentToken() == Token.START_ARRAY) { parser.nextToken(); while (this.parser.currentToken() != Token.END_ARRAY) { - isChildrenValueValid |= parseToken(path, currentFieldName); + isChildrenValueValid |= parseToken(path); } this.parser.nextToken(); - } else if (this.parser.currentToken() == Token.END_ARRAY) { - // skip } else if (this.parser.currentToken() == Token.START_OBJECT) { parser.nextToken(); while (this.parser.currentToken() != Token.END_OBJECT) { - isChildrenValueValid |= parseToken(path, currentFieldName); + isChildrenValueValid |= parseToken(path); } this.parser.nextToken(); } else { @@ -148,21 +144,6 @@ public void removeKeyOfNullValue() { this.keyList.remove(keyList.size() - 1); } - private boolean processNoNestedValue() throws IOException { - if (parser.currentToken() == Token.VALUE_NULL) { - return true; - } else if (this.parser.currentToken() == Token.VALUE_STRING - || this.parser.currentToken() == Token.VALUE_NUMBER - || this.parser.currentToken() == Token.VALUE_BOOLEAN) { - String value = this.parser.textOrNull(); - if (value != null) { - this.valueList.add(value); - } - return true; - } - return false; - } - private String parseValue() throws IOException { switch (this.parser.currentToken()) { case VALUE_BOOLEAN: @@ -172,7 +153,7 @@ private String parseValue() throws IOException { return this.parser.textOrNull(); // Handle other token types as needed default: - throw new IOException("Unsupported value token type [" + parser.currentToken() + "]"); + throw new ParsingException(parser.getTokenLocation(), "Unexpected value token type [" + parser.currentToken() + "]"); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index bf8f83e1b95df..738efcfafdca1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -30,6 +30,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.xcontent.JsonToStringXContentParser; +import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -568,31 +569,41 @@ protected void parseCreateField(ParseContext context) throws IOException { if (context.externalValueSet()) { String value = context.externalValue().toString(); parseValueAddFields(context, value, fieldType().name()); - } else if (context.parser().currentToken() != XContentParser.Token.VALUE_NULL) { - JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.IGNORE_DEPRECATIONS, - context.parser(), - fieldType().name() - ); - /* - JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser - It reads the JSON object and parsed to a list of string - */ - XContentParser parser = jsonToStringParser.parseObject(); - - XContentParser.Token currentToken; - while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - switch (currentToken) { - case FIELD_NAME: - fieldName = parser.currentName(); - break; - case VALUE_STRING: - String value = parser.textOrNull(); - parseValueAddFields(context, value, fieldName); - break; + } else { + XContentParser ctxParser = context.parser(); + if (ctxParser.currentToken() != XContentParser.Token.VALUE_NULL) { + if (ctxParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + ctxParser.getTokenLocation(), + "[" + this.name() + "] unexpected token [" + ctxParser.currentToken() + "] in flat_object field value" + ); } + JsonToStringXContentParser jsonToStringParser = new JsonToStringXContentParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + ctxParser, + fieldType().name() + ); + /* + JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser + It reads the JSON object and parsed to a list of string + */ + XContentParser parser = jsonToStringParser.parseObject(); + + XContentParser.Token currentToken; + while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + switch (currentToken) { + case FIELD_NAME: + fieldName = parser.currentName(); + break; + case VALUE_STRING: + String value = parser.textOrNull(); + parseValueAddFields(context, value, fieldName); + break; + } + + } } } From 036f6bc836442a35e5f1cab47345e3209e04d085 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Sat, 21 Sep 2024 09:43:27 +0530 Subject: [PATCH 131/357] Add disallow settings update during repository in use ITs (#16001) Signed-off-by: Ashish Singh --- .../snapshots/ConcurrentSnapshotsIT.java | 55 +++++++++++++++++++ .../AbstractSnapshotIntegTestCase.java | 10 ++++ 2 files changed, 65 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java index 15e92f6f7204b..26f8ab5dd8d09 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java @@ -50,6 +50,7 @@ import org.opensearch.common.util.concurrent.UncategorizedExecutionException; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.discovery.AbstractDisruptionTestCase; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoryData; @@ -135,6 +136,60 @@ public void testLongRunningSnapshotAllowsConcurrentSnapshot() throws Exception { assertSuccessful(createSlowFuture); } + public void testSettingsUpdateFailWhenCreateSnapshotInProgress() throws Exception { + // Start a cluster with a cluster manager node and a data node + internalCluster().startClusterManagerOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + // Create a repository with random settings + Settings.Builder settings = randomRepositorySettings(); + createRepository(repoName, "mock", settings); + createIndexWithContent("index"); + // Start a full snapshot and block it on the data node + final ActionFuture createSlowFuture = startFullSnapshotBlockedOnDataNode( + "slow-snapshot", + repoName, + dataNode + ); + Thread.sleep(1000); // Wait for the snapshot to start + assertFalse(createSlowFuture.isDone()); // Ensure the snapshot is still in progress + // Attempt to update the repository settings while the snapshot is in progress + IllegalStateException ex = assertThrows(IllegalStateException.class, () -> updateRepository(repoName, "mock", settings)); + // Verify that the update fails with an appropriate exception + assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); + unblockNode(repoName, dataNode); // Unblock the snapshot + assertSuccessful(createSlowFuture); // Ensure the snapshot completes successfully + } + + public void testSettingsUpdateFailWhenDeleteSnapshotInProgress() throws InterruptedException { + // Start a cluster with a cluster manager node and a data node + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + // Create a repository with random settings + Settings.Builder settings = randomRepositorySettings(); + createRepository(repoName, "mock", settings); + createIndexWithContent("index"); + final String snapshotName = "snapshot-1"; + // Create a full snapshot + SnapshotInfo snapshotInfo = createFullSnapshot(repoName, snapshotName); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); // Ensure the snapshot was successful + assertEquals(RestStatus.OK, snapshotInfo.status()); // Ensure the snapshot status is OK + // Start deleting the snapshot and block it on the cluster manager node + ActionFuture future = deleteSnapshotBlockedOnClusterManager(repoName, snapshotName); + Thread.sleep(1000); // Wait for the delete operation to start + assertFalse(future.isDone()); // Ensure the delete operation is still in progress + // Attempt to update the repository settings while the delete operation is in progress + IllegalStateException ex = assertThrows( + IllegalStateException.class, + () -> updateRepository(repoName, "mock", randomRepositorySettings()) + ); + // Verify that the update fails with an appropriate exception + assertEquals("trying to modify or unregister repository that is currently used", ex.getMessage()); + unblockNode(repoName, clusterManagerName); // Unblock the delete operation + assertAcked(future.actionGet()); // Wait for the delete operation to complete + } + public void testDeletesAreBatched() throws Exception { internalCluster().startClusterManagerOnlyNode(); final String dataNode = internalCluster().startDataOnlyNode(); diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index a8bb10fe20752..0bfa70a771f65 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -665,6 +665,16 @@ public void onTimeout(TimeValue timeout) { } } + protected ActionFuture deleteSnapshotBlockedOnClusterManager(String repoName, String snapshotName) { + blockClusterManagerFromDeletingIndexNFile(repoName); + return deleteSnapshot(repoName, snapshotName); + } + + protected ActionFuture deleteSnapshot(String repoName, String snapshotName) { + logger.info("--> Deleting snapshot [{}] to repo [{}]", snapshotName, repoName); + return clusterAdmin().prepareDeleteSnapshot(repoName, snapshotName).execute(); + } + protected ActionFuture startFullSnapshotBlockedOnDataNode(String snapshotName, String repoName, String dataNode) throws InterruptedException { blockDataNode(repoName, dataNode); From 13433670227eb6037cc5b743a78099864da9f61c Mon Sep 17 00:00:00 2001 From: Shubh Sahu Date: Mon, 23 Sep 2024 12:38:39 +0530 Subject: [PATCH 132/357] Fixing flaky test testRemoteIndexPathFileExistsAfterMigration (#15977) Signed-off-by: Shubh Sahu --- .../RemoteMigrationIndexMetadataUpdateIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java index b55219e1cb37f..35689deeb8134 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java @@ -19,7 +19,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.index.remote.RemoteIndexPath; -import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.InternalTestCluster; @@ -533,17 +532,18 @@ public void testRemoteIndexPathFileExistsAfterMigration() throws Exception { // validate remote index path file exists logger.info("---> Asserting remote index path file exists"); - String fileNamePrefix = String.join(RemoteIndexPathUploader.DELIMITER, indexUUID, "7", RemoteIndexPath.DEFAULT_VERSION); assertTrue(FileSystemUtils.exists(translogRepoPath.resolve(RemoteIndexPath.DIR))); Path[] files = FileSystemUtils.files(translogRepoPath.resolve(RemoteIndexPath.DIR)); assertEquals(1, files.length); - assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix))); + logger.info(files[0].toString()); + assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID))); assertTrue(FileSystemUtils.exists(segmentRepoPath.resolve(RemoteIndexPath.DIR))); files = FileSystemUtils.files(segmentRepoPath.resolve(RemoteIndexPath.DIR)); assertEquals(1, files.length); - assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileNamePrefix))); + logger.info(files[0].toString()); + assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(indexUUID))); } /** From b984b9f56f9b0f9dbb5322bddf04f3b45940b730 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:54:47 -0500 Subject: [PATCH 133/357] Bump actions/github-script from 5 to 7 (#16039) * Bump actions/github-script from 5 to 7 Bumps [actions/github-script](https://github.com/actions/github-script) from 5 to 7. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v5...v7) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/delete_backport_branch.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 91626a0bc5092..1c73d3d250ce3 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -12,7 +12,7 @@ jobs: if: startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch - uses: actions/github-script@v5 + uses: actions/github-script@v7 with: script: | github.rest.git.deleteRef({ diff --git a/CHANGELOG.md b/CHANGELOG.md index ae0b0e0e7a235..69a054acd24d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) ### Changed From c85ee6898b4e3e2f4a2037e42337a7eb8033de9e Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 24 Sep 2024 07:29:17 +0800 Subject: [PATCH 134/357] Fix search_as_you_type not supporting multi-fields (#15988) * Fix search_as_you_type not supporting multi-fields Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong * Fix test failure Signed-off-by: Gao Binlong * Add more yaml test Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../mapper/SearchAsYouTypeFieldMapper.java | 13 +++- .../SearchAsYouTypeFieldMapperTests.java | 14 ++++ ...filtering.yml => 380_bitmap_filtering.yml} | 0 .../test/search/390_search_as_you_type.yml | 75 +++++++++++++++++++ 5 files changed, 101 insertions(+), 2 deletions(-) rename rest-api-spec/src/main/resources/rest-api-spec/test/search/{370_bitmap_filtering.yml => 380_bitmap_filtering.yml} (100%) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a054acd24d2..c1041cdf4f64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) - Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) diff --git a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java index 366e848416328..f08815ebbbd1e 100644 --- a/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -264,7 +264,15 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); - return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this); + return new SearchAsYouTypeFieldMapper( + name, + ft, + multiFieldsBuilder.build(this, context), + copyTo.build(), + prefixFieldMapper, + shingleFieldMappers, + this + ); } } @@ -623,12 +631,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew public SearchAsYouTypeFieldMapper( String simpleName, SearchAsYouTypeFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder ) { - super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, multiFields, copyTo); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index f55ad2e9d659c..7746cb714a019 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -298,6 +298,20 @@ private void assertMultiField(int shingleSize) throws IOException { } } + public void testSubField() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping( + b -> b.field("type", "search_as_you_type") + .startObject("fields") + .startObject("subField") + .field("type", "keyword") + .endObject() + .endObject() + ) + ); + assertThat(mapperService.fieldType("field.subField"), instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + public void testIndexOptions() throws IOException { DocumentMapper mapper = createDocumentMapper( fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets")) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml rename to rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml new file mode 100644 index 0000000000000..689b3edfd1066 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml @@ -0,0 +1,75 @@ +setup: + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + text: + type: search_as_you_type + fields: + subField: + type: keyword + - do: + index: + index: test_1 + id: 1 + body: { text: test search as you type } + + - do: + indices.refresh: + index: [test_1] + +--- +teardown: + - do: + indices.delete: + index: test_1 + +# related issue: https://github.com/opensearch-project/OpenSearch/issues/5035 +--- +"Test search_as_you_type data type supports multi-fields": + - skip: + version: " - 2.99.99" + reason: "the bug was fixed since 3.0.0" + + - do: + indices.get_mapping: { + index: test_1 + } + + - match: {test_1.mappings.properties.text.type: search_as_you_type} + - match: {test_1.mappings.properties.text.fields.subField.type: keyword} + + - do: + search: + index: test_1 + body: + query: + multi_match: + query: "test search" + type: "bool_prefix" + + - match: {hits.total.value: 1} + + - do: + search: + index: test_1 + body: + query: + multi_match: + query: "test search" + type: "bool_prefix" + fields: ["text.subField"] + + - match: {hits.total.value: 1} + + - do: + search: + index: test_1 + body: + query: + term: + text.subField: "test search as you type" + + - match: {hits.total.value: 1} From eeb2f3997bb84f33f13b848e125051ecf2c2a1c7 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Tue, 24 Sep 2024 11:07:30 +0530 Subject: [PATCH 135/357] Fix flaky test in RemoteStoreMigrationSettingsUpdateIT (#16048) Signed-off-by: Ashish Singh --- .../RemoteStoreMigrationSettingsUpdateIT.java | 7 +++++-- .../java/org/opensearch/test/OpenSearchIntegTestCase.java | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java index 30c597e405f4e..d9e72dd137182 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -68,7 +69,6 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() assertRemoteStoreBackedIndex(indexName2); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15793") public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception { logger.info("Initialize cluster: gives non remote cluster manager"); initializeCluster(false); @@ -76,7 +76,10 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix logger.info("Add remote and non-remote nodes"); setClusterMode(MIXED.mode); addRemote = false; - String nonRemoteNodeName = internalCluster().startNode(); + Settings settings = Settings.builder() + .put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "") + .build(); + String nonRemoteNodeName = internalCluster().startNode(settings); addRemote = true; String remoteNodeName = internalCluster().startNode(); internalCluster().validateClusterFormed(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index e474ef202b235..68a2b8086a92e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -403,7 +403,7 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private static Boolean segmentsPathFixedPrefix; - private static Boolean snapshotShardPathFixedPrefix; + protected static Boolean snapshotShardPathFixedPrefix; private Path remoteStoreRepositoryPath; @@ -2904,7 +2904,7 @@ private static Settings buildRemoteStoreNodeAttributes( settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean()); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : ""); settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : ""); - settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), segmentsPathFixedPrefix ? "c" : ""); + settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : ""); return settings.build(); } From d6bda7dca75dc4d328b61abcd21b35a6bdfcab05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 15:08:24 +0800 Subject: [PATCH 136/357] Bump dnsjava:dnsjava from 3.6.1 to 3.6.2 in /test/fixtures/hdfs-fixture (#16041) * Bump dnsjava:dnsjava from 3.6.1 to 3.6.2 in /test/fixtures/hdfs-fixture Bumps [dnsjava:dnsjava](https://github.com/dnsjava/dnsjava) from 3.6.1 to 3.6.2. - [Release notes](https://github.com/dnsjava/dnsjava/releases) - [Changelog](https://github.com/dnsjava/dnsjava/blob/master/Changelog) - [Commits](https://github.com/dnsjava/dnsjava/compare/v3.6.1...v3.6.2) --- updated-dependencies: - dependency-name: dnsjava:dnsjava dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1041cdf4f64d..8a32f10ebc691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) +- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 20fd1058a181d..21fe28bfd835d 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -55,7 +55,7 @@ dependencies { exclude group: 'com.nimbusds' exclude module: "commons-configuration2" } - api "dnsjava:dnsjava:3.6.1" + api "dnsjava:dnsjava:3.6.2" api "org.codehaus.jettison:jettison:${versions.jettison}" api "org.apache.commons:commons-compress:${versions.commonscompress}" api "commons-codec:commons-codec:${versions.commonscodec}" From 848e5c676c780d11be78784da87359ae77f47f50 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 24 Sep 2024 22:35:30 +0800 Subject: [PATCH 137/357] Update version check in yaml test file for the bug fix of avoid infinite loop in flat_object parsing (#16057) Signed-off-by: Gao Binlong --- .../main/resources/rest-api-spec/test/index/90_flat_object.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index c0fc0090abedf..e8da81d7bee41 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -76,7 +76,7 @@ teardown: --- "Invalid docs": - skip: - version: "- 2.99.99" + version: "- 2.17.99" reason: "parsing of these objects would infinite loop prior to 2.18" # The following documents are invalid. - do: From 6a2911962539c6288a7f126d529e2393c792fdeb Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 24 Sep 2024 12:49:31 -0400 Subject: [PATCH 138/357] Add support for docker compose v2 in TestFixturesPlugin (#16049) * Update references for docker-compose to docker compose Signed-off-by: Craig Perkins * Support both Signed-off-by: Craig Perkins * Check if V2 is available Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Update CHANGELOG text Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../gradle/docker/DockerSupportService.java | 11 +++++++++++ .../gradle/testfixtures/TestFixturesPlugin.java | 9 +++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a32f10ebc691..dfec81c070bab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) ### Changed +- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) ### Deprecated diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java index fc78792bb3551..77d7997d6d48d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java @@ -106,6 +106,7 @@ public DockerAvailability getDockerAvailability() { Version version = null; boolean isVersionHighEnough = false; boolean isComposeAvailable = false; + boolean isComposeV2Available = false; // Check if the Docker binary exists final Optional dockerBinary = getDockerPath(); @@ -129,6 +130,8 @@ public DockerAvailability getDockerAvailability() { if (lastResult.isSuccess() && composePath.isPresent()) { isComposeAvailable = runCommand(composePath.get(), "version").isSuccess(); } + + isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess(); } } } @@ -138,6 +141,7 @@ public DockerAvailability getDockerAvailability() { this.dockerAvailability = new DockerAvailability( isAvailable, isComposeAvailable, + isComposeV2Available, isVersionHighEnough, dockerPath, version, @@ -356,6 +360,11 @@ public static class DockerAvailability { */ public final boolean isComposeAvailable; + /** + * True if docker compose is available. + */ + public final boolean isComposeV2Available; + /** * True if the installed Docker version is >= 17.05 */ @@ -379,6 +388,7 @@ public static class DockerAvailability { DockerAvailability( boolean isAvailable, boolean isComposeAvailable, + boolean isComposeV2Available, boolean isVersionHighEnough, String path, Version version, @@ -386,6 +396,7 @@ public static class DockerAvailability { ) { this.isAvailable = isAvailable; this.isComposeAvailable = isComposeAvailable; + this.isComposeV2Available = isComposeV2Available; this.isVersionHighEnough = isVersionHighEnough; this.path = path; this.version = version; diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java index e8772522b19a4..f65e231cd2e50 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java @@ -171,7 +171,11 @@ public void execute(Task task) { .findFirst(); composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker"); - composeExtension.getUseDockerComposeV2().set(false); + if (dockerSupport.get().getDockerAvailability().isComposeV2Available) { + composeExtension.getUseDockerComposeV2().set(true); + } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) { + composeExtension.getUseDockerComposeV2().set(false); + } tasks.named("composeUp").configure(t -> { // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions @@ -228,7 +232,8 @@ private void maybeSkipTask(Provider dockerSupport, TaskPro private void maybeSkipTask(Provider dockerSupport, Task task) { task.onlyIf(spec -> { - boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeAvailable; + boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available + || dockerSupport.get().getDockerAvailability().isComposeAvailable; if (isComposeAvailable == false) { LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath()); } From b3cc802a11a5685bd32652caecca7e396b00e021 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:24:00 +0530 Subject: [PATCH 139/357] Change the version to 2.18 as term-check fallback is merged to 2.x (#16062) Signed-off-by: Rajiv Kumar Vaidyanathan --- .../support/clustermanager/term/GetTermVersionResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java index cf8e6085d4b32..f3fd756c6db4b 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java @@ -41,7 +41,7 @@ public GetTermVersionResponse(ClusterStateTermVersion clusterStateTermVersion, b public GetTermVersionResponse(StreamInput in) throws IOException { super(in); this.clusterStateTermVersion = new ClusterStateTermVersion(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { this.isStatePresentInRemote = in.readOptionalBoolean(); } else { this.isStatePresentInRemote = false; @@ -51,7 +51,7 @@ public GetTermVersionResponse(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { clusterStateTermVersion.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalBoolean(isStatePresentInRemote); } } From 12dadcf2aff9ce7e8f1bf4d6afe022684a7d6496 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Wed, 25 Sep 2024 08:35:11 +0530 Subject: [PATCH 140/357] Retry remote state download while bootstrap (#15950) * Retry remote state download while bootstrap Signed-off-by: Sooraj Sinha --- .../RemoteStoreClusterStateRestoreIT.java | 4 +- .../opensearch/gateway/GatewayMetaState.java | 60 +++++++++++-- .../GatewayMetaStatePersistedStateTests.java | 84 ++++++++++++++++++- 3 files changed, 134 insertions(+), 14 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index d078ba05faa12..6a5adf5ea4fb7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -42,6 +42,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.Before; +import java.io.IOError; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -339,10 +340,11 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh for (UploadedIndexMetadata md : manifest.getIndices()) { Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/")); } + internalCluster().stopAllNodes(); } catch (IOException e) { throw new RuntimeException(e); } - assertThrows(IllegalStateException.class, () -> addNewNodes(dataNodeCount, clusterManagerNodeCount)); + assertThrows(IOError.class, () -> internalCluster().client()); // Test is complete // Starting a node without remote state to ensure test cleanup diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index b3836edcd7d6c..f5da6df2689bd 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -66,13 +66,13 @@ import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.model.RemoteClusterStateManifestInfo; import org.opensearch.index.recovery.RemoteStoreRestoreService; -import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; import org.opensearch.node.Node; import org.opensearch.plugins.MetadataUpgrader; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.Closeable; +import java.io.IOError; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -109,6 +109,8 @@ public class GatewayMetaState implements Closeable { */ public static final String STALE_STATE_CONFIG_NODE_ID = "STALE_STATE_CONFIG"; + private final Logger logger = LogManager.getLogger(GatewayMetaState.class); + private PersistedStateRegistry persistedStateRegistry; public PersistedState getPersistedState() { @@ -175,15 +177,11 @@ public void start( ); if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) { // Load state from remote - final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore( - // Remote Metadata should always override local disk Metadata - // if local disk Metadata's cluster uuid is UNKNOWN_UUID - ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(), - lastKnownClusterUUID, - false, - new String[] {} + clusterState = restoreClusterStateWithRetries( + remoteStoreRestoreService, + clusterState, + lastKnownClusterUUID ); - clusterState = remoteRestoreResult.getClusterState(); } } remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID); @@ -258,6 +256,50 @@ public void start( } } + private ClusterState restoreClusterStateWithRetries( + RemoteStoreRestoreService remoteStoreRestoreService, + ClusterState clusterState, + String lastKnownClusterUUID + ) { + int maxAttempts = 5; + int delayInMills = 200; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + logger.info("Attempt {} to restore cluster state", attempt); + return restoreClusterState(remoteStoreRestoreService, clusterState, lastKnownClusterUUID); + } catch (Exception e) { + if (attempt == maxAttempts) { + // Throw an Error so that the process is halted. + throw new IOError(e); + } + try { + TimeUnit.MILLISECONDS.sleep(delayInMills); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); // Restore interrupted status + throw new RuntimeException(ie); + } + delayInMills = delayInMills * 2; + } + } + // This statement will never be reached. + return null; + } + + ClusterState restoreClusterState( + RemoteStoreRestoreService remoteStoreRestoreService, + ClusterState clusterState, + String lastKnownClusterUUID + ) { + return remoteStoreRestoreService.restore( + // Remote Metadata should always override local disk Metadata + // if local disk Metadata's cluster uuid is UNKNOWN_UUID + ClusterState.builder(clusterState).metadata(Metadata.EMPTY_METADATA).build(), + lastKnownClusterUUID, + false, + new String[] {} + ).getClusterState(); + } + // exposed so it can be overridden by tests ClusterState prepareInitialClusterState(TransportService transportService, ClusterService clusterService, ClusterState clusterState) { assert clusterState.nodes().getLocalNode() == null : "prepareInitialClusterState must only be called once"; diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 5ea5241762753..efdb3076f419c 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -1244,14 +1244,72 @@ public void testGatewayForRemoteStateForInitialBootstrapBlocksApplied() throws I } } - private MockGatewayMetaState newGatewayForRemoteState( + public void testGatewayMetaStateRemoteStateDownloadRetries() throws IOException { + MockGatewayMetaState gateway = null; + MockGatewayMetaState gatewayMetaStateSpy = null; + try { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid"); + RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class); + when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow( + new IllegalStateException("unable to download cluster state") + ).thenReturn(RemoteRestoreResult.build("test-cluster-uuid", null, ClusterState.EMPTY_STATE)); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + gateway = initializeGatewayForRemoteState(true); + gatewayMetaStateSpy = Mockito.spy(gateway); + startGatewayForRemoteState( + gatewayMetaStateSpy, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + ClusterState.EMPTY_STATE + ); + verify(gatewayMetaStateSpy, times(2)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any()); + } finally { + IOUtils.close(gatewayMetaStateSpy); + } + } + + public void testGatewayMetaStateRemoteStateDownloadFailure() throws IOException { + MockGatewayMetaState gateway = null; + final MockGatewayMetaState gatewayMetaStateSpy; + try { + RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); + when(remoteClusterStateService.getLastKnownUUIDFromRemote("test-cluster")).thenReturn("test-cluster-uuid"); + RemoteStoreRestoreService remoteStoreRestoreService = mock(RemoteStoreRestoreService.class); + when(remoteStoreRestoreService.restore(any(), any(), anyBoolean(), any())).thenThrow( + new IllegalStateException("unable to download cluster state") + ); + final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); + gateway = initializeGatewayForRemoteState(true); + gatewayMetaStateSpy = Mockito.spy(gateway); + assertThrows( + Error.class, + () -> startGatewayForRemoteState( + gatewayMetaStateSpy, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + ClusterState.EMPTY_STATE + ) + ); + verify(gatewayMetaStateSpy, times(5)).restoreClusterState(Mockito.any(), Mockito.any(), Mockito.any()); + } finally { + IOUtils.close(gateway); + } + } + + private MockGatewayMetaState initializeGatewayForRemoteState(boolean prepareFullState) { + return new MockGatewayMetaState(localNode, bigArrays, prepareFullState); + } + + private MockGatewayMetaState startGatewayForRemoteState( + MockGatewayMetaState gateway, RemoteClusterStateService remoteClusterStateService, RemoteStoreRestoreService remoteStoreRestoreService, PersistedStateRegistry persistedStateRegistry, - ClusterState currentState, - boolean prepareFullState + ClusterState currentState ) throws IOException { - MockGatewayMetaState gateway = new MockGatewayMetaState(localNode, bigArrays, prepareFullState); String randomRepoName = "randomRepoName"; String stateRepoTypeAttributeKey = String.format( Locale.getDefault(), @@ -1305,6 +1363,24 @@ private MockGatewayMetaState newGatewayForRemoteState( return gateway; } + private MockGatewayMetaState newGatewayForRemoteState( + RemoteClusterStateService remoteClusterStateService, + RemoteStoreRestoreService remoteStoreRestoreService, + PersistedStateRegistry persistedStateRegistry, + ClusterState currentState, + boolean prepareFullState + ) throws IOException { + MockGatewayMetaState gatewayMetaState = initializeGatewayForRemoteState(prepareFullState); + startGatewayForRemoteState( + gatewayMetaState, + remoteClusterStateService, + remoteStoreRestoreService, + persistedStateRegistry, + currentState + ); + return gatewayMetaState; + } + private static BigArrays getBigArrays() { return usually() ? BigArrays.NON_RECYCLING_INSTANCE From dc4dbced2f9046ab5debb5dfec4e9634217be0c2 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 25 Sep 2024 15:27:25 +0530 Subject: [PATCH 141/357] Optimize remote store GC flow with pinned timestamps (#15943) Signed-off-by: Sachin Kale --- ...rePinnedTimestampsGarbageCollectionIT.java | 433 ++++++++++++++++++ .../snapshots/DeleteSnapshotV2IT.java | 231 +++++----- .../store/RemoteSegmentStoreDirectory.java | 16 +- .../RemoteFsTimestampAwareTranslog.java | 171 ++++--- .../index/translog/RemoteFsTranslog.java | 13 +- .../transfer/TranslogTransferMetadata.java | 10 + .../RemoteStorePinnedTimestampService.java | 4 + .../blobstore/BlobStoreRepository.java | 43 +- .../snapshots/SnapshotsService.java | 10 +- .../RemoteSegmentStoreDirectoryTests.java | 3 +- .../RemoteFsTimestampAwareTranslogTests.java | 147 +++++- 11 files changed, 854 insertions(+), 227 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java new file mode 100644 index 0000000000000..0a2668c60d3bd --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java @@ -0,0 +1,433 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.translog.transfer.TranslogTransferMetadata; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING; +import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; +import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteStorePinnedTimestampsGarbageCollectionIT extends RemoteStoreBaseIntegTestCase { + static final String INDEX_NAME = "remote-store-test-idx-1"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false) + .build(); + } + + private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException { + long currentTime = System.currentTimeMillis(); + int maxRetry = 10; + while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) { + Thread.sleep(1000); + maxRetry--; + } + } + + ActionListener noOpActionListener = new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }; + + public void testLiveIndexNoPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(2, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testLiveIndexNoPinnedTimestampsWithExtraGenSettingWithinLimit() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 10) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 9); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertTrue(metadataFiles.size() >= numDocs + 1); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }, 30, TimeUnit.SECONDS); + } + + public void testLiveIndexNoPinnedTimestampsWithExtraGenSetting() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = 5; + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(5, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testLiveIndexWithPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(3, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + public void testIndexDeletionNoPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(2, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + + keepPinnedTimestampSchedulerUpdated(); + client().admin().indices().prepareDelete(INDEX_NAME).get(); + + assertBusy(() -> { + assertEquals(0, Files.list(translogMetadataPath).collect(Collectors.toList()).size()); + assertEquals(0, Files.list(translogDataPath).collect(Collectors.toList()).size()); + }); + } + + public void testIndexDeletionWithPinnedTimestamps() throws Exception { + prepareCluster(1, 1, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(0, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 0) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + String shardMetadataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + METADATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogMetadataPath = Path.of(translogRepoPath + "/" + shardMetadataPath); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(3, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }, 30, TimeUnit.SECONDS); + + keepPinnedTimestampSchedulerUpdated(); + client().admin().indices().prepareDelete(INDEX_NAME).get(); + + assertBusy(() -> { + List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); + assertEquals(1, metadataFiles.size()); + + verifyTranslogDataFileCount(metadataFiles, translogDataPath); + }); + } + + private void verifyTranslogDataFileCount(List metadataFiles, Path translogDataPath) throws IOException { + List mdFiles = metadataFiles.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList()); + Set generations = new HashSet<>(); + for (String mdFile : mdFiles) { + Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(mdFile); + generations.addAll(LongStream.rangeClosed(minMaxGen.v1(), minMaxGen.v2()).boxed().collect(Collectors.toList())); + } + assertEquals(generations.size() * 2, Files.list(translogDataPath).collect(Collectors.toList()).size()); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index 1d7a58384c0be..c4e3a478c8540 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -23,19 +23,31 @@ import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class DeleteSnapshotV2IT extends AbstractSnapshotIntegTestCase { private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + private void keepPinnedTimestampSchedulerUpdated() throws InterruptedException { + long currentTime = System.currentTimeMillis(); + int maxRetry = 10; + while (maxRetry > 0 && RemoteStorePinnedTimestampService.getPinnedTimestamps().v1() <= currentTime) { + Thread.sleep(1000); + maxRetry--; + } + } + public void testDeleteShallowCopyV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); @@ -74,8 +86,8 @@ public void testDeleteShallowCopyV2() throws Exception { createIndex(indexName1, getRemoteStoreBackedIndexSettings()); createIndex(indexName2, getRemoteStoreBackedIndexSettings()); - final int numDocsInIndex1 = 10; - final int numDocsInIndex2 = 20; + final int numDocsInIndex1 = 1; + final int numDocsInIndex2 = 2; indexRandomDocs(indexName1, numDocsInIndex1); indexRandomDocs(indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); @@ -92,7 +104,7 @@ public void testDeleteShallowCopyV2() throws Exception { assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); createIndex(indexName3, getRemoteStoreBackedIndexSettings()); - indexRandomDocs(indexName3, 10); + indexRandomDocs(indexName3, 1); CreateSnapshotResponse createSnapshotResponse2 = client().admin() .cluster() .prepareCreateSnapshot(snapshotRepoName, snapshotName2) @@ -105,109 +117,101 @@ public void testDeleteShallowCopyV2() throws Exception { assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); assertAcked(client().admin().indices().prepareDelete(indexName1)); - Thread.sleep(100); - AcknowledgedResponse deleteResponse = client().admin() - .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotName2) - .setSnapshots(snapshotName2) - .get(); + AcknowledgedResponse deleteResponse = client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, snapshotName2).get(); assertTrue(deleteResponse.isAcknowledged()); // test delete non-existent snapshot assertThrows( SnapshotMissingException.class, - () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() + () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").get() ); - } - public void testDeleteShallowCopyV2MultipleSnapshots() throws Exception { + public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); + Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); + settings = Settings.builder() + .put(settings) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNode(settings); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(2); - internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); - - String indexName1 = "testindex1"; - String indexName2 = "testindex2"; - String indexName3 = "testindex3"; - String snapshotRepoName = "test-create-snapshot-repo"; - String snapshotName1 = "test-create-snapshot1"; - String snapshotName2 = "test-create-snapshot2"; - Path absolutePath1 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); - - Client client = client(); - - assertAcked( - client.admin() - .cluster() - .preparePutRepository(snapshotRepoName) - .setType(FsRepository.TYPE) - .setSettings( - Settings.builder() - .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) - .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) - .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) - .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) - .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) - ) + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + clusterManagerName ); - createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath)); - createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, 25); - final int numDocsInIndex1 = 10; - final int numDocsInIndex2 = 20; - indexRandomDocs(indexName1, numDocsInIndex1); - indexRandomDocs(indexName2, numDocsInIndex2); - ensureGreen(indexName1, indexName2); + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); + logger.info("--> create two remote index shallow snapshots"); CreateSnapshotResponse createSnapshotResponse = client().admin() .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .prepareCreateSnapshot(snapshotRepoName, "snap1") .setWaitForCompletion(true) .get(); - SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo(); - createIndex(indexName3, getRemoteStoreBackedIndexSettings()); - indexRandomDocs(indexName3, 10); + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); - CreateSnapshotResponse createSnapshotResponse2 = client().admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .setWaitForCompletion(true) - .get(); - snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); + // delete remote store index + assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); + + logger.info("--> delete snapshot 1"); + + Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog"); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); - AcknowledgedResponse deleteResponse = client().admin() + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + keepPinnedTimestampSchedulerUpdated(); + + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotName1, snapshotName2) - .setSnapshots(snapshotName2) + .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName()) .get(); - assertTrue(deleteResponse.isAcknowledged()); + assertAcked(deleteSnapshotResponse); - // test delete non-existent snapshot - assertThrows( - SnapshotMissingException.class, - () -> client().admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "random-snapshot").setSnapshots(snapshotName2).get() - ); + // Delete is async. Give time for it + assertBusy(() -> { + try { + assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath)); + } catch (NoSuchFileException e) { + fail(); + } + }, 60, TimeUnit.SECONDS); + assertBusy(() -> { + try { + assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(translogPath)); + } catch (NoSuchFileException e) { + fail(); + } + }, 60, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15692") - public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exception { + public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2MultipleSnapshots() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); @@ -242,11 +246,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .get() .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); - String numShards = client().admin() - .indices() - .prepareGetSettings(remoteStoreEnabledIndexName) - .get() - .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_NUMBER_OF_SHARDS); + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); + + Path segmentsPath = Path.of(String.valueOf(shardPath), "segments", "data"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1"); logger.info("--> create two remote index shallow snapshots"); CreateSnapshotResponse createSnapshotResponse = client().admin() @@ -256,6 +260,11 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .get(); SnapshotInfo snapshotInfo1 = createSnapshotResponse.getSnapshotInfo(); + List segmentsPostSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList()); + List translogPostSnapshot1 = Files.list(translogPath).collect(Collectors.toList()); + + forceMerge(1); + refresh(remoteStoreEnabledIndexName); indexRandomDocs(remoteStoreEnabledIndexName, 25); CreateSnapshotResponse createSnapshotResponse2 = client().admin() @@ -264,46 +273,28 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio .setWaitForCompletion(true) .get(); SnapshotInfo snapshotInfo2 = createSnapshotResponse2.getSnapshotInfo(); + + List segmentsPostSnapshot2 = Files.list(segmentsPath).collect(Collectors.toList()); + List translogPostSnapshot2 = Files.list(translogPath).collect(Collectors.toList()); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); assertThat(snapshotInfo2.snapshotId().getName(), equalTo("snap2")); - // delete remote store index - assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); - - logger.info("--> delete snapshot 2"); + assertBusy(() -> assertTrue(translogPostSnapshot2.size() > translogPostSnapshot1.size()), 60, TimeUnit.SECONDS); + assertBusy(() -> assertTrue(segmentsPostSnapshot2.size() > segmentsPostSnapshot1.size()), 60, TimeUnit.SECONDS); - Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); - Path shardPath = Path.of(String.valueOf(indexPath), "0"); - Path segmentsPath = Path.of(String.valueOf(shardPath), "segments"); - Path translogPath = Path.of(String.valueOf(shardPath), "translog"); - - // Get total segments remote store directory file count for deleted index and shard 0 - int segmentFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); - int translogFilesCountBeforeDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(translogPath); - - RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); - AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() - .cluster() - .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo2.snapshotId().getName()) - .get(); - assertAcked(deleteSnapshotResponse); - - Thread.sleep(5000); - - assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountBeforeDeletingSnapshot1)); - } catch (Exception e) {} - }, 30, TimeUnit.SECONDS); - int segmentFilesCountAfterDeletingSnapshot1 = RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath); + // delete remote store index + assertAcked(client().admin().indices().prepareDelete(remoteStoreEnabledIndexName)); logger.info("--> delete snapshot 1"); RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); // on snapshot deletion, remote store segment files should get cleaned up for deleted index - `remote-index-1` - deleteSnapshotResponse = clusterManagerClient.admin() + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() .cluster() .prepareDeleteSnapshot(snapshotRepoName, snapshotInfo1.snapshotId().getName()) .get(); @@ -311,23 +302,21 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2() throws Exceptio // Delete is async. Give time for it assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(segmentsPath), lessThan(segmentFilesCountAfterDeletingSnapshot1)); - } catch (Exception e) {} + List segmentsPostDeletionOfSnapshot1 = Files.list(segmentsPath).collect(Collectors.toList()); + assertTrue(segmentsPostDeletionOfSnapshot1.size() < segmentsPostSnapshot2.size()); }, 60, TimeUnit.SECONDS); - - assertBusy(() -> { - try { - assertThat(RemoteStoreBaseIntegTestCase.getFileCount(translogPath), lessThan(translogFilesCountBeforeDeletingSnapshot1)); - } catch (Exception e) {} - }, 60, TimeUnit.SECONDS); - + // To uncomment following, we need to handle deletion of generations in translog cleanup flow + // List translogPostDeletionOfSnapshot1 = Files.list(translogPath).collect(Collectors.toList()); + // Delete is async. Give time for it + // assertBusy(() -> assertEquals(translogPostSnapshot2.size() - translogPostSnapshot1.size(), + // translogPostDeletionOfSnapshot1.size()), 60, TimeUnit.SECONDS); } private Settings snapshotV2Settings(Path remoteStoreRepoPath) { Settings settings = Settings.builder() .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), false) .build(); return settings; } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 72bf07d4b03b2..5be516166803e 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -854,7 +854,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException } // Check last fetch status of pinned timestamps. If stale, return. - if (RemoteStoreUtils.isPinnedTimestampStateStale()) { + if (lastNMetadataFilesToKeep != 0 && RemoteStoreUtils.isPinnedTimestampStateStale()) { logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); return; } @@ -994,7 +994,8 @@ public static void remoteDirectoryCleanup( String remoteStoreRepoForIndex, String indexUUID, ShardId shardId, - RemoteStorePathStrategy pathStrategy + RemoteStorePathStrategy pathStrategy, + boolean forceClean ) { try { RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory( @@ -1003,8 +1004,12 @@ public static void remoteDirectoryCleanup( shardId, pathStrategy ); - remoteSegmentStoreDirectory.deleteStaleSegments(0); - remoteSegmentStoreDirectory.deleteIfEmpty(); + if (forceClean) { + remoteSegmentStoreDirectory.delete(); + } else { + remoteSegmentStoreDirectory.deleteStaleSegments(0); + remoteSegmentStoreDirectory.deleteIfEmpty(); + } } catch (Exception e) { staticLogger.error("Exception occurred while deleting directory", e); } @@ -1023,7 +1028,10 @@ private boolean deleteIfEmpty() throws IOException { logger.info("Remote directory still has files, not deleting the path"); return false; } + return delete(); + } + private boolean delete() { try { remoteDataDirectory.delete(); remoteMetadataDirectory.delete(); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 27d34ec0d05af..e61a9606175ee 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; import java.util.function.LongConsumer; @@ -61,6 +62,7 @@ public class RemoteFsTimestampAwareTranslog extends RemoteFsTranslog { private final Map> oldFormatMetadataFileGenerationMap; private final Map> oldFormatMetadataFilePrimaryTermMap; private final AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + private final AtomicBoolean triggerTrimOnMinRemoteGenReferencedChange = new AtomicBoolean(false); public RemoteFsTimestampAwareTranslog( TranslogConfig config, @@ -105,6 +107,11 @@ protected void onDelete() { } } + @Override + protected void onMinRemoteGenReferencedChange() { + triggerTrimOnMinRemoteGenReferencedChange.set(true); + } + @Override public void trimUnreferencedReaders() throws IOException { trimUnreferencedReaders(false, true); @@ -135,14 +142,22 @@ protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. - if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) { + if (indexDeleted == false && (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get())) { return; } // This is to fail fast and avoid listing md files un-necessarily. if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale"); + return; + } + + // This code block ensures parity with RemoteFsTranslog. Without this, we will end up making list translog metadata + // call in each invocation of trimUnreferencedReaders + if (indexDeleted == false && triggerTrimOnMinRemoteGenReferencedChange.get() == false) { return; + } else if (triggerTrimOnMinRemoteGenReferencedChange.get()) { + triggerTrimOnMinRemoteGenReferencedChange.set(false); } // Since remote generation deletion is async, this ensures that only one generation deletion happens at a time. @@ -158,7 +173,7 @@ public void onResponse(List blobMetadata) { List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); try { - if (metadataFiles.size() <= 1) { + if (indexDeleted == false && metadataFiles.size() <= 1) { logger.debug("No stale translog metadata files found"); remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); return; @@ -166,16 +181,12 @@ public void onResponse(List blobMetadata) { // Check last fetch status of pinned timestamps. If stale, return. if (indexDeleted == false && RemoteStoreUtils.isPinnedTimestampStateStale()) { - logger.warn("Skipping remote segment store garbage collection as last fetch of pinned timestamp is stale"); + logger.warn("Skipping remote translog garbage collection as last fetch of pinned timestamp is stale"); remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); return; } - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted( - metadataFiles, - metadataFilePinnedTimestampMap, - logger - ); + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted); // If index is not deleted, make sure to keep latest metadata file if (indexDeleted == false) { @@ -194,10 +205,11 @@ public void onResponse(List blobMetadata) { metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); logger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + Set generationsToBeDeleted = getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - indexDeleted + indexDeleted ? Long.MAX_VALUE : getMinGenerationToKeepInRemote() ); logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted); @@ -208,7 +220,11 @@ public void onResponse(List blobMetadata) { generationsToBeDeleted, remoteGenerationDeletionPermits::release ); + } else { + remoteGenerationDeletionPermits.release(); + } + if (metadataFilesToBeDeleted.isEmpty() == false) { // Delete stale metadata files translogTransferManager.deleteMetadataFilesAsync( metadataFilesToBeDeleted, @@ -217,11 +233,10 @@ public void onResponse(List blobMetadata) { // Update cache to keep only those metadata files that are not getting deleted oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); - // Delete stale primary terms deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted); } else { - remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + remoteGenerationDeletionPermits.release(); } } catch (Exception e) { remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); @@ -237,18 +252,16 @@ public void onFailure(Exception e) { translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); } + private long getMinGenerationToKeepInRemote() { + return minRemoteGenReferenced - indexSettings().getRemoteTranslogExtraKeep(); + } + // Visible for testing protected Set getGenerationsToBeDeleted( List metadataFilesNotToBeDeleted, List metadataFilesToBeDeleted, - boolean indexDeleted + long minGenerationToKeepInRemote ) throws IOException { - long maxGenerationToBeDeleted = Long.MAX_VALUE; - - if (indexDeleted == false) { - maxGenerationToBeDeleted = minRemoteGenReferenced - 1 - indexSettings().getRemoteTranslogExtraKeep(); - } - Set generationsFromMetadataFilesToBeDeleted = new HashSet<>(); for (String mdFile : metadataFilesToBeDeleted) { Tuple minMaxGen = getMinMaxTranslogGenerationFromMetadataFile(mdFile, translogTransferManager); @@ -262,21 +275,31 @@ protected Set getGenerationsToBeDeleted( Set generationsToBeDeleted = new HashSet<>(); for (long generation : generationsFromMetadataFilesToBeDeleted) { // Check if the generation is not referred by metadata file matching pinned timestamps - if (generation <= maxGenerationToBeDeleted && isGenerationPinned(generation, pinnedGenerations) == false) { + // The check with minGenerationToKeep is redundant but kept as to make sure we don't delete generations + // that are not persisted in remote segment store yet. + if (generation < minGenerationToKeepInRemote && isGenerationPinned(generation, pinnedGenerations) == false) { generationsToBeDeleted.add(generation); } } return generationsToBeDeleted; } - protected List getMetadataFilesToBeDeleted(List metadataFiles) { - return getMetadataFilesToBeDeleted(metadataFiles, metadataFilePinnedTimestampMap, logger); + protected List getMetadataFilesToBeDeleted(List metadataFiles, boolean indexDeleted) { + return getMetadataFilesToBeDeleted( + metadataFiles, + metadataFilePinnedTimestampMap, + getMinGenerationToKeepInRemote(), + indexDeleted, + logger + ); } // Visible for testing protected static List getMetadataFilesToBeDeleted( List metadataFiles, Map metadataFilePinnedTimestampMap, + long minGenerationToKeepInRemote, + boolean indexDeleted, Logger logger ) { Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); @@ -312,6 +335,22 @@ protected static List getMetadataFilesToBeDeleted( metadataFilesToBeDeleted.size() ); + if (indexDeleted == false) { + // Filter out metadata files based on minGenerationToKeep + List metadataFilesContainingMinGenerationToKeep = metadataFilesToBeDeleted.stream().filter(md -> { + long maxGeneration = TranslogTransferMetadata.getMaxGenerationFromFileName(md); + return maxGeneration == -1 || maxGeneration >= minGenerationToKeepInRemote; + }).collect(Collectors.toList()); + metadataFilesToBeDeleted.removeAll(metadataFilesContainingMinGenerationToKeep); + + logger.trace( + "metadataFilesContainingMinGenerationToKeep.size = {}, metadataFilesToBeDeleted based on minGenerationToKeep filtering = {}, minGenerationToKeep = {}", + metadataFilesContainingMinGenerationToKeep.size(), + metadataFilesToBeDeleted.size(), + minGenerationToKeepInRemote + ); + } + return metadataFilesToBeDeleted; } @@ -472,50 +511,60 @@ protected static Tuple getMinMaxPrimaryTermFromMetadataFile( } } - public static void cleanup(TranslogTransferManager translogTransferManager) throws IOException { - ActionListener> listMetadataFilesListener = new ActionListener<>() { - @Override - public void onResponse(List blobMetadata) { - List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + public static void cleanupOfDeletedIndex(TranslogTransferManager translogTransferManager, boolean forceClean) throws IOException { + if (forceClean) { + translogTransferManager.delete(); + } else { + ActionListener> listMetadataFilesListener = new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + List metadataFiles = blobMetadata.stream().map(BlobMetadata::name).collect(Collectors.toList()); + + try { + if (metadataFiles.isEmpty()) { + staticLogger.debug("No stale translog metadata files found"); + return; + } + List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + true, // This method gets called when the index is no longer present + staticLogger + ); + if (metadataFilesToBeDeleted.isEmpty()) { + staticLogger.debug("No metadata files to delete"); + return; + } + staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); - try { - if (metadataFiles.isEmpty()) { - staticLogger.debug("No stale translog metadata files found"); - return; - } - List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), staticLogger); - if (metadataFilesToBeDeleted.isEmpty()) { - staticLogger.debug("No metadata files to delete"); - return; - } - staticLogger.debug(() -> "metadataFilesToBeDeleted = " + metadataFilesToBeDeleted); + // For all the files that we are keeping, fetch min and max generations + List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); + staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); - // For all the files that we are keeping, fetch min and max generations - List metadataFilesNotToBeDeleted = new ArrayList<>(metadataFiles); - metadataFilesNotToBeDeleted.removeAll(metadataFilesToBeDeleted); - staticLogger.debug(() -> "metadataFilesNotToBeDeleted = " + metadataFilesNotToBeDeleted); + // Delete stale metadata files + translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); - // Delete stale metadata files - translogTransferManager.deleteMetadataFilesAsync(metadataFilesToBeDeleted, () -> {}); + // Delete stale primary terms + deleteStaleRemotePrimaryTerms( + metadataFilesNotToBeDeleted, + translogTransferManager, + new HashMap<>(), + new AtomicLong(Long.MAX_VALUE), + staticLogger + ); + } catch (Exception e) { + staticLogger.error("Exception while cleaning up metadata and primary terms", e); + } + } - // Delete stale primary terms - deleteStaleRemotePrimaryTerms( - metadataFilesNotToBeDeleted, - translogTransferManager, - new HashMap<>(), - new AtomicLong(Long.MAX_VALUE), - staticLogger - ); - } catch (Exception e) { + @Override + public void onFailure(Exception e) { staticLogger.error("Exception while cleaning up metadata and primary terms", e); } - } - - @Override - public void onFailure(Exception e) { - staticLogger.error("Exception while cleaning up metadata and primary terms", e); - } - }; - translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + }; + translogTransferManager.listTranslogMetadataFilesAsync(listMetadataFilesListener); + } } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 812852d107682..85f58f898826f 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -683,12 +683,17 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen @Override public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOException { maxRemoteTranslogGenerationUploaded = generation; + long previousMinRemoteGenReferenced = minRemoteGenReferenced; minRemoteGenReferenced = getMinFileGeneration(); + if (previousMinRemoteGenReferenced != minRemoteGenReferenced) { + onMinRemoteGenReferencedChange(); + } logger.debug( - "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}", + "Successfully uploaded translog for primary term = {}, generation = {}, maxSeqNo = {}, minRemoteGenReferenced = {}", primaryTerm, generation, - maxSeqNo + maxSeqNo, + minRemoteGenReferenced ); } @@ -702,6 +707,10 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) thro } } + protected void onMinRemoteGenReferencedChange() { + + } + @Override public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) { return minSeqNoToKeep; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 3b8885055e8f7..7fe3305545085 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -170,6 +170,16 @@ public static Tuple getMinMaxTranslogGenerationFromFilename(String f } } + public static long getMaxGenerationFromFileName(String filename) { + String[] tokens = filename.split(METADATA_SEPARATOR); + try { + return RemoteStoreUtils.invertLong(tokens[2]); + } catch (Exception e) { + logger.error(() -> new ParameterizedMessage("Exception while getting max generation from: {}", filename), e); + return -1; + } + } + public static Tuple getMinMaxPrimaryTermFromFilename(String filename) { String[] tokens = filename.split(METADATA_SEPARATOR); if (tokens.length < 7) { diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 71133615ed056..1448c46583f6a 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -240,6 +240,10 @@ public void unpinTimestamp(long timestamp, String pinningEntity, ActionListener< } } + public void forceSyncPinnedTimestamps() { + asyncUpdatePinnedTimestampTask.run(); + } + @Override public void close() throws IOException { asyncUpdatePinnedTimestampTask.close(); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 0292cecc36a81..14c201e819994 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -199,7 +199,6 @@ import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS; -import static org.opensearch.snapshots.SnapshotsService.SNAPSHOT_PINNED_TIMESTAMP_DELIMITER; /** * BlobStore - based implementation of Snapshot Repository @@ -1309,7 +1308,7 @@ private void cleanUpRemoteStoreFilesForDeletedIndicesV2( } // iterate through all the indices and trigger remote store directory cleanup for deleted index segments for (String indexId : uniqueIndexIds) { - cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory); + cleanRemoteStoreDirectoryIfNeeded(snapshotIds, indexId, repositoryData, remoteSegmentStoreDirectoryFactory, false); } afterCleanupsListener.onResponse(null); } catch (Exception e) { @@ -1357,11 +1356,17 @@ private void removeSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.unpinTimestamp( timestampToUnpin, - repository + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotId.getUUID(), + SnapshotsService.getPinningEntity(repository, snapshotId.getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { - logger.debug("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName()); + logger.info("Timestamp {} unpinned successfully for snapshot {}", timestampToUnpin, snapshotId.getName()); + try { + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); + logger.debug("Successfully synced pinned timestamp state"); + } catch (Exception e) { + logger.warn("Exception while updating pinning timestamp state, snapshot deletion will continue", e); + } listener.onResponse(null); } @@ -1466,7 +1471,8 @@ public static void remoteDirectoryCleanupAsync( String indexUUID, ShardId shardId, String threadPoolName, - RemoteStorePathStrategy pathStrategy + RemoteStorePathStrategy pathStrategy, + boolean forceClean ) { threadpool.executor(threadPoolName) .execute( @@ -1476,7 +1482,8 @@ public static void remoteDirectoryCleanupAsync( remoteStoreRepoForIndex, indexUUID, shardId, - pathStrategy + pathStrategy, + forceClean ), indexUUID, shardId @@ -1532,7 +1539,8 @@ protected void releaseRemoteStoreLockAndCleanup( indexUUID, new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt(shardId)), ThreadPool.Names.REMOTE_PURGE, - remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy() + remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy(), + false ); } } @@ -2095,7 +2103,7 @@ private void executeOneStaleIndexDelete( deleteResult = deleteResult.add(cleanUpStaleSnapshotShardPathsFile(matchingShardPaths, snapshotShardPaths)); if (remoteSegmentStoreDirectoryFactory != null) { - cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory); + cleanRemoteStoreDirectoryIfNeeded(deletedSnapshots, indexSnId, oldRepoData, remoteSegmentStoreDirectoryFactory, true); } // Finally, we delete the [base_path]/indexId folder @@ -2167,7 +2175,8 @@ private void cleanRemoteStoreDirectoryIfNeeded( Collection deletedSnapshots, String indexSnId, RepositoryData oldRepoData, - RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory + RemoteSegmentStoreDirectoryFactory remoteSegmentStoreDirectoryFactory, + boolean forceClean ) { assert (indexSnId != null); @@ -2219,9 +2228,16 @@ private void cleanRemoteStoreDirectoryIfNeeded( prevIndexMetadata.getIndexUUID(), shard, ThreadPool.Names.REMOTE_PURGE, - remoteStorePathStrategy + remoteStorePathStrategy, + forceClean + ); + remoteTranslogCleanupAsync( + remoteTranslogRepository, + shard, + remoteStorePathStrategy, + prevIndexMetadata, + forceClean ); - remoteTranslogCleanupAsync(remoteTranslogRepository, shard, remoteStorePathStrategy, prevIndexMetadata); } } } catch (Exception e) { @@ -2245,7 +2261,8 @@ private void remoteTranslogCleanupAsync( Repository remoteTranslogRepository, ShardId shardId, RemoteStorePathStrategy remoteStorePathStrategy, - IndexMetadata prevIndexMetadata + IndexMetadata prevIndexMetadata, + boolean forceClean ) { assert remoteTranslogRepository instanceof BlobStoreRepository; boolean indexMetadataEnabled = RemoteStoreUtils.determineTranslogMetadataEnabled(prevIndexMetadata); @@ -2262,7 +2279,7 @@ private void remoteTranslogCleanupAsync( indexMetadataEnabled ); try { - RemoteFsTimestampAwareTranslog.cleanup(translogTransferManager); + RemoteFsTimestampAwareTranslog.cleanupOfDeletedIndex(translogTransferManager, forceClean); } catch (IOException e) { logger.error("Exception while cleaning up remote translog for shard: " + shardId, e); } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index f6e550525a3e5..23f6deff3715d 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -613,7 +613,7 @@ private void updateSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.pinTimestamp( timestampToPin, - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(), + getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { @@ -631,6 +631,10 @@ public void onFailure(Exception e) { ); } + public static String getPinningEntity(String repositoryName, String snapshotUUID) { + return repositoryName + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotUUID; + } + private void cloneSnapshotPinnedTimestamp( RepositoryData repositoryData, SnapshotId sourceSnapshot, @@ -640,8 +644,8 @@ private void cloneSnapshotPinnedTimestamp( ) { remoteStorePinnedTimestampService.cloneTimestamp( timestampToPin, - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + sourceSnapshot.getUUID(), - snapshot.getRepository() + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshot.getSnapshotId().getUUID(), + getPinningEntity(snapshot.getRepository(), sourceSnapshot.getUUID()), + getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()), new ActionListener() { @Override public void onResponse(Void unused) { diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index ecd6620dbea15..df3df81361a12 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -565,7 +565,8 @@ public void testCleanupAsync() throws Exception { repositoryName, indexUUID, shardId, - pathStrategy + pathStrategy, + false ); verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathStrategy); verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 4ec68f7fb75b4..e6871414cf5e0 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -214,6 +214,7 @@ public void onFailure(Exception e) { // Old format metadata file String oldFormatMdFilename = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"; assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(oldFormatMdFilename)); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(oldFormatMdFilename)); // Node id containing separator String nodeIdWithSeparator = @@ -221,10 +222,14 @@ public void onFailure(Exception e) { Tuple minMaxGen = TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(nodeIdWithSeparator); Long minGen = Long.MAX_VALUE - 9223372036438563958L; assertEquals(minGen, minMaxGen.v1()); + Long maxGen = Long.MAX_VALUE - 9223372036854774799L; + assertEquals(maxGen, minMaxGen.v2()); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(nodeIdWithSeparator)); // Malformed md filename String malformedMdFileName = "metadata__9223372036438563903__9223372036854774799__9223370311919910393__node1__xyz__3__1"; assertNull(TranslogTransferMetadata.getMinMaxTranslogGenerationFromFilename(malformedMdFileName)); + assertEquals(Long.MAX_VALUE - 9223372036854774799L, TranslogTransferMetadata.getMaxGenerationFromFileName(malformedMdFileName)); } public void testGetMinMaxPrimaryTermFromFilename() throws Exception { @@ -330,43 +335,60 @@ public void testSimpleOperationsUpload() throws Exception { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); - addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); assertBusy(() -> { assertEquals( - 16, + 10, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); }); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); - RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); // Fetch pinned timestamps so that it won't be stale updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(3); + translog.trimUnreferencedReaders(); - translog.setMinSeqNoToKeep(4); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(6); translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + assertEquals(1, translog.readers.size()); + assertBusy(() -> { + assertEquals(2, translog.allUploaded().size()); + assertEquals(4, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 16, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 })); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 })); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); // Fetch pinned timestamps so that it won't be stale updatePinnedTimstampTask.run(); translog.trimUnreferencedReaders(); - assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); - assertEquals(5, translog.readers.size()); + + assertEquals(3, translog.readers.size()); assertBusy(() -> { - assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); - assertEquals(10, translog.allUploaded().size()); + assertEquals(6, translog.allUploaded().size()); + assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); assertEquals( - 10, + 12, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); - }, 60, TimeUnit.SECONDS); + }, 30, TimeUnit.SECONDS); } @Override @@ -397,7 +419,7 @@ public void testMetadataFileDeletion() throws Exception { ); updatePinnedTimstampTask.run(); translog.trimUnreferencedReaders(); - assertBusy(() -> { assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); }); + assertBusy(() -> { assertEquals(3, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); }); } public void testMetadataFileDeletionWithPinnedTimestamps() throws Exception { @@ -568,7 +590,7 @@ public void testDrainSync() throws Exception { assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertEquals(1, translog.readers.size()); assertBusy(() -> assertEquals(2, translog.allUploaded().size())); - assertBusy(() -> assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); + assertBusy(() -> assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size())); } @Override @@ -647,7 +669,7 @@ public void testGetGenerationsToBeDeletedEmptyMetadataFilesNotToBeDeleted() thro Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - true + Long.MAX_VALUE ); Set md1Generations = LongStream.rangeClosed(4, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 37).boxed().collect(Collectors.toSet()); @@ -683,7 +705,7 @@ public void testGetGenerationsToBeDeleted() throws IOException { Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( metadataFilesNotToBeDeleted, metadataFilesToBeDeleted, - true + Long.MAX_VALUE ); Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); @@ -708,7 +730,10 @@ public void testGetMetadataFilesToBeDeletedNoExclusion() { "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1" ); - assertEquals(metadataFiles, ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles)); + assertEquals( + metadataFiles, + RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), Long.MAX_VALUE, false, logger) + ); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { @@ -724,7 +749,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); } @@ -746,7 +777,13 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(2, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); @@ -769,11 +806,77 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throw "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" ); - List metadataFilesToBeDeleted = ((RemoteFsTimestampAwareTranslog) translog).getMetadataFilesToBeDeleted(metadataFiles); + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + Long.MAX_VALUE, + false, + logger + ); assertEquals(1, metadataFilesToBeDeleted.size()); assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0)); } + public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + when(blobContainer.listBlobs()).thenReturn(Map.of()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + // MaxGen 7 + "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", + // MaxGen 12 + "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 10 + "metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + 10L, + false, + logger + ); + assertEquals(2, metadataFilesToBeDeleted.size()); + assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); + } + + public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex() throws IOException { + long currentTimeInMillis = System.currentTimeMillis(); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + + when(blobContainer.listBlobs()).thenReturn(Map.of()); + + updatePinnedTimstampTask.run(); + + List metadataFiles = List.of( + // MaxGen 7 + "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", + // MaxGen 12 + "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 17 + "metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1" + ); + + List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( + metadataFiles, + new HashMap<>(), + 10L, + true, + logger + ); + assertEquals(metadataFiles, metadataFilesToBeDeleted); + } + public void testIsGenerationPinned() { TreeSet> pinnedGenerations = new TreeSet<>(new TreeSet<>((o1, o2) -> { if (Objects.equals(o1.v1(), o2.v1()) == false) { From 0617d95c0e3185c4d5dd38fc9d1637da1b11b0ab Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 25 Sep 2024 18:16:48 +0530 Subject: [PATCH 142/357] Modify flaky test (#15854) Signed-off-by: Shivansh Arora --- .../gateway/remote/RemoteStatePublicationIT.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index ffb9352e8ba47..578c922c80a0d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -235,7 +235,6 @@ public void testRemotePublicationDownloadStats() { assertDataNodeDownloadStats(nodesStatsResponseDataNode); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15767") public void testRemotePublicationDisabledByRollingRestart() throws Exception { prepareCluster(3, 2, INDEX_NAME, 1, 2); ensureStableCluster(5); @@ -272,7 +271,6 @@ public void doAfterNodes(int n, Client client) { assertTrue( stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0 ); - assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); } else { DiscoveryStats stats = nodeStats.getDiscoveryStats(); assertEquals(0, stats.getPublishStats().getFullClusterStateReceivedCount()); @@ -297,7 +295,7 @@ public void doAfterNodes(int n, Client client) { ); if (activeCMRestarted) { assertNull(remoteState.getLastAcceptedState()); - // assertNull(remoteState.getLastAcceptedManifest()); + assertNull(remoteState.getLastAcceptedManifest()); } else { ClusterState localState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL) .getLastAcceptedState(); @@ -326,7 +324,6 @@ public void doAfterNodes(int n, Client client) { response.getNodes().forEach(nodeStats -> { PublishClusterStateStats stats = nodeStats.getDiscoveryStats().getPublishStats(); assertTrue(stats.getFullClusterStateReceivedCount() > 0 || stats.getCompatibleClusterStateDiffReceivedCount() > 0); - assertEquals(0, stats.getIncompatibleClusterStateDiffReceivedCount()); }); NodesInfoResponse nodesInfoResponse = client().admin() .cluster() @@ -341,7 +338,7 @@ public void doAfterNodes(int n, Client client) { PersistedStateRegistry registry = internalCluster().getInstance(PersistedStateRegistry.class, node); CoordinationState.PersistedState remoteState = registry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE); assertNull(remoteState.getLastAcceptedState()); - // assertNull(remoteState.getLastAcceptedManifest()); + assertNull(remoteState.getLastAcceptedManifest()); }); } From f1acc7aad7db4c3c9ce2e0ac331b02105ddc85f5 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Wed, 25 Sep 2024 18:45:36 +0530 Subject: [PATCH 143/357] Fix issue of red index on close for remote enabled clusters (#15990) * Fix red index on close for remote translog Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- .../opensearch/remotestore/RemoteStoreIT.java | 69 ++++++++++++++++- .../index/translog/RemoteFsTranslog.java | 21 ++++- .../index/translog/RemoteFsTranslogTests.java | 77 +++++++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 692727357a88a..ebb911c739eb3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -18,6 +18,7 @@ import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchPhaseExecutionException; +import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; @@ -202,7 +203,7 @@ public void testRemoteTranslogCleanup() throws Exception { public void testStaleCommitDeletionWithInvokeFlush() throws Exception { String dataNode = internalCluster().startNode(); - createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000L, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); String segmentsPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.get(getNodeSettings()); @@ -1011,4 +1012,70 @@ public void testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates() throws .get() ); } + + public void testCloseIndexWithNoOpSyncAndFlushForSyncTranslog() throws InterruptedException { + internalCluster().startNodes(3); + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "5s")) + .get(); + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(0, 10000L, -1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s"); + createIndex(INDEX_NAME, settings.build()); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + if (randomBoolean()) { + for (int i = 0; i < randomIntBetween(1, 5); i++) { + indexSingleDoc(INDEX_NAME); + } + flushAndRefresh(INDEX_NAME); + } + // Index single doc to start the asyn io processor to run which will lead to 10s wait time before the next sync. + indexSingleDoc(INDEX_NAME); + // Reduce the latch for the main thread to flush after some sleep. + latch.countDown(); + // Index another doc and in this case the flush would have happened before the sync. + indexSingleDoc(INDEX_NAME); + }).start(); + // Wait for atleast one doc to be ingested. + latch.await(); + // Sleep for some time for the next doc to be present in lucene buffer. If flush happens first before the doc #2 + // gets indexed, then it goes into the happy case where the close index happens succefully. + Thread.sleep(1000); + // Flush so that the subsequent sync or flushes are no-op. + flush(INDEX_NAME); + // Closing the index involves translog.sync and shard.flush which are now no-op. + client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet(); + Thread.sleep(10000); + ensureGreen(INDEX_NAME); + } + + public void testCloseIndexWithNoOpSyncAndFlushForAsyncTranslog() throws InterruptedException { + internalCluster().startNodes(3); + Settings.Builder settings = Settings.builder() + .put(remoteStoreIndexSettings(0, 10000L, -1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "1s") + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Durability.ASYNC) + .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "10s"); + createIndex(INDEX_NAME, settings.build()); + CountDownLatch latch = new CountDownLatch(1); + new Thread(() -> { + // Index some docs to start the asyn io processor to run which will lead to 10s wait time before the next sync. + indexSingleDoc(INDEX_NAME); + indexSingleDoc(INDEX_NAME); + indexSingleDoc(INDEX_NAME); + // Reduce the latch for the main thread to flush after some sleep. + latch.countDown(); + }).start(); + // Wait for atleast one doc to be ingested. + latch.await(); + // Flush so that the subsequent sync or flushes are no-op. + flush(INDEX_NAME); + // Closing the index involves translog.sync and shard.flush which are now no-op. + client().admin().indices().close(Requests.closeIndexRequest(INDEX_NAME)).actionGet(); + Thread.sleep(10000); + ensureGreen(INDEX_NAME); + } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 85f58f898826f..80760ed6ddf12 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -78,6 +78,9 @@ public class RemoteFsTranslog extends Translog { // min generation referred by last uploaded translog protected volatile long minRemoteGenReferenced; + // the max global checkpoint that has been synced + protected volatile long globalCheckpointSynced; + // clean up translog folder uploaded by previous primaries once protected final SetOnce olderPrimaryCleaned = new SetOnce<>(); @@ -437,9 +440,10 @@ private boolean upload(long primaryTerm, long generation, long maxSeqNo) throws config.getNodeId() ).build() ) { + Checkpoint checkpoint = current.getLastSyncedCheckpoint(); return translogTransferManager.transferSnapshot( transferSnapshotProvider, - new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo) + new RemoteFsTranslogTransferListener(generation, primaryTerm, maxSeqNo, checkpoint.globalCheckpoint) ); } finally { syncPermit.release(SYNC_PERMIT); @@ -474,7 +478,10 @@ public void sync() throws IOException { public boolean syncNeeded() { try (ReleasableLock lock = readLock.acquire()) { return current.syncNeeded() - || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0); + || (maxRemoteTranslogGenerationUploaded + 1 < this.currentFileGeneration() && current.totalOperations() == 0) + // The below condition on GCP exists to handle global checkpoint updates during close index. + // Refer issue - https://github.com/opensearch-project/OpenSearch/issues/15989 + || (current.getLastSyncedCheckpoint().globalCheckpoint > globalCheckpointSynced); } } @@ -674,10 +681,13 @@ private class RemoteFsTranslogTransferListener implements TranslogTransferListen private final long maxSeqNo; - RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo) { + private final long globalCheckpoint; + + RemoteFsTranslogTransferListener(long generation, long primaryTerm, long maxSeqNo, long globalCheckpoint) { this.generation = generation; this.primaryTerm = primaryTerm; this.maxSeqNo = maxSeqNo; + this.globalCheckpoint = globalCheckpoint; } @Override @@ -685,6 +695,11 @@ public void onUploadComplete(TransferSnapshot transferSnapshot) throws IOExcepti maxRemoteTranslogGenerationUploaded = generation; long previousMinRemoteGenReferenced = minRemoteGenReferenced; minRemoteGenReferenced = getMinFileGeneration(); + // Update the global checkpoint only if the supplied global checkpoint is greater than it + // When a new writer is created the + if (globalCheckpoint > globalCheckpointSynced) { + globalCheckpointSynced = globalCheckpoint; + } if (previousMinRemoteGenReferenced != minRemoteGenReferenced) { onMinRemoteGenReferencedChange(); } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 339d876274557..03c77a9a83f57 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -1801,6 +1801,83 @@ public void testDownloadWithEmptyTranslogOnlyInLocal() throws IOException { assertArrayEquals(filesPostFirstDownload, filesPostSecondDownload); } + public void testSyncWithGlobalCheckpointUpdate() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 2 })); + + // Set a global checkpoint + long initialGlobalCheckpoint = 1L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Verify that the globalCheckpointSynced is updated + assertEquals(initialGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + + // Update global checkpoint + long newGlobalCheckpoint = 2L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Add a new operation and sync + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 2, primaryTerm.get(), new byte[] { 3 })); + translog.sync(); + + // Verify that the globalCheckpointSynced is updated to the new value + assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + } + + public void testSyncNeededWithGlobalCheckpointUpdate() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + + // Set initial global checkpoint + long initialGlobalCheckpoint = 0L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Verify that sync is not needed + assertFalse(translog.syncNeeded()); + + // Update global checkpoint + long newGlobalCheckpoint = 1L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Verify that sync is now needed due to global checkpoint update + assertTrue(translog.syncNeeded()); + + // Sync again + translog.sync(); + + // Verify that sync is not needed after syncing + assertFalse(translog.syncNeeded()); + } + + public void testGlobalCheckpointUpdateDuringClose() throws IOException { + ArrayList ops = new ArrayList<>(); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); + + // Set initial global checkpoint + long initialGlobalCheckpoint = 0L; + globalCheckpoint.set(initialGlobalCheckpoint); + + // Sync the translog + translog.sync(); + + // Update global checkpoint + long newGlobalCheckpoint = 1L; + globalCheckpoint.set(newGlobalCheckpoint); + + // Close the translog + translog.close(); + + // Verify that the last synced checkpoint includes the updated global checkpoint + assertEquals(newGlobalCheckpoint, ((RemoteFsTranslog) translog).getLastSyncedCheckpoint().globalCheckpoint); + } + public class ThrowingBlobRepository extends FsRepository { private final Environment environment; From a42e51ddb93d50a9bb0455bb6bda9fe461fe8f5b Mon Sep 17 00:00:00 2001 From: Finn Date: Wed, 25 Sep 2024 16:30:37 -0700 Subject: [PATCH 144/357] Fix segment count for optimized agg rest test (#16067) Signed-off-by: Finn Carroll --- .../search.aggregation/360_date_histogram.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml index 8c8a98b2db22c..0cabbf6f25133 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml @@ -109,6 +109,19 @@ setup: --- "Date histogram aggregation w/ shared field range test": + - do: + indices.create: + index: dhisto-agg-w-query + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + refresh_interval: -1 + mappings: + properties: + date: + type: date + - do: bulk: refresh: true @@ -127,6 +140,11 @@ setup: - '{"index": {}}' - '{"date": "2025-02-14"}' + - do: + indices.forcemerge: + index: dhisto-agg-w-query + max_num_segments: 1 + - do: search: index: dhisto-agg-w-query From daf1669b7ca77c62fc82ba9162d3708702b13fce Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Wed, 25 Sep 2024 18:16:59 -0700 Subject: [PATCH 145/357] Added support for msearch API to pass search pipeline name (#15923) * Added support for search pipeline name in multi search API Signed-off-by: Owais * Updated CHANGELOG Signed-off-by: Owais * Pulled search pipeline in MultiSearchRequest and updated test Signed-off-by: Owais * Updated test Signed-off-by: Owais * Updated SearchRequest with search pipeline from source Signed-off-by: Owais * Added tests for parseSearchRequest Signed-off-by: Owais * Guard serialization with version check Signed-off-by: Owais * Updated version and added another test for serialization Signed-off-by: Owais --------- Signed-off-by: Owais --- CHANGELOG.md | 1 + .../action/search/MultiSearchRequest.java | 4 ++ .../rest/action/search/RestSearchAction.java | 2 +- .../search/builder/SearchSourceBuilder.java | 36 +++++++++++- .../action/search/SearchRequestTests.java | 24 ++++++-- .../builder/SearchSourceBuilderTests.java | 21 +++++++ .../pipeline/SearchPipelineServiceTests.java | 58 +++++++++++++++++++ 7 files changed, 139 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfec81c070bab..935901d410178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) +- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java index 5b887b48f696e..f16d7d1e7d6a3 100644 --- a/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/MultiSearchRequest.java @@ -310,6 +310,10 @@ public static void readMultiLineFormat( ) { consumer.accept(searchRequest, parser); } + + if (searchRequest.source() != null && searchRequest.source().pipeline() != null) { + searchRequest.pipeline(searchRequest.source().pipeline()); + } // move pointers from = nextMarker + 1; } diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java index 3a6b45013e892..05465e32631fd 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java @@ -210,7 +210,7 @@ public static void parseSearchRequest( searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); - searchRequest.pipeline(request.param("search_pipeline")); + searchRequest.pipeline(request.param("search_pipeline", searchRequest.source().pipeline())); checkRestTotalHits(request, searchRequest); request.paramAsBoolean(INCLUDE_NAMED_QUERIES_SCORE_PARAM, false); diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index 8a9704b04566f..dd4e4d073cb1b 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -224,6 +224,8 @@ public static HighlightBuilder highlight() { private Map searchPipelineSource = null; + private String searchPipeline; + /** * Constructs a new search source builder. */ @@ -297,6 +299,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { derivedFields = in.readList(DerivedField::new); } } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + searchPipeline = in.readOptionalString(); + } } @Override @@ -377,6 +382,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(derivedFields); } } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(searchPipeline); + } } /** @@ -1111,6 +1119,13 @@ public Map searchPipelineSource() { return searchPipelineSource; } + /** + * @return a search pipeline name defined within the search source (see {@link org.opensearch.search.pipeline.SearchPipelineService}) + */ + public String pipeline() { + return searchPipeline; + } + /** * Define a search pipeline to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}. */ @@ -1119,6 +1134,14 @@ public SearchSourceBuilder searchPipelineSource(Map searchPipeli return this; } + /** + * Define a search pipeline name to process this search request and/or its response. See {@link org.opensearch.search.pipeline.SearchPipelineService}. + */ + public SearchSourceBuilder pipeline(String searchPipeline) { + this.searchPipeline = searchPipeline; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1216,6 +1239,7 @@ private SearchSourceBuilder shallowCopy( rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; rewrittenBuilder.derivedFieldsObject = derivedFieldsObject; rewrittenBuilder.derivedFields = derivedFields; + rewrittenBuilder.searchPipeline = searchPipeline; return rewrittenBuilder; } @@ -1283,6 +1307,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th sort(parser.text()); } else if (PROFILE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { profile = parser.booleanValue(); + } else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) { + searchPipeline = parser.text(); } else { throw new ParsingException( parser.getTokenLocation(), @@ -1612,6 +1638,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t } + if (searchPipeline != null) { + builder.field(SEARCH_PIPELINE.getPreferredName(), searchPipeline); + } + return builder; } @@ -1889,7 +1919,8 @@ public int hashCode() { trackTotalHitsUpTo, pointInTimeBuilder, derivedFieldsObject, - derivedFields + derivedFields, + searchPipeline ); } @@ -1934,7 +1965,8 @@ public boolean equals(Object obj) { && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) && Objects.equals(derivedFieldsObject, other.derivedFieldsObject) - && Objects.equals(derivedFields, other.derivedFields); + && Objects.equals(derivedFields, other.derivedFields) + && Objects.equals(searchPipeline, other.searchPipeline); } @Override diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index 40514c526f190..acda1445bacbb 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -42,6 +42,8 @@ import org.opensearch.geometry.LinearRing; import org.opensearch.index.query.GeoShapeQueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.search.RestSearchAction; import org.opensearch.search.AbstractSearchTestCase; import org.opensearch.search.Scroll; import org.opensearch.search.builder.PointInTimeBuilder; @@ -50,14 +52,18 @@ import org.opensearch.search.rescore.QueryRescorerBuilder; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; +import org.opensearch.test.rest.FakeRestRequest; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.IntConsumer; import static java.util.Collections.emptyMap; +import static org.opensearch.action.search.SearchType.DFS_QUERY_THEN_FETCH; import static org.opensearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; public class SearchRequestTests extends AbstractSearchTestCase { @@ -242,6 +248,19 @@ public void testCopyConstructor() throws IOException { assertNotSame(deserializedRequest, searchRequest); } + public void testParseSearchRequestWithUnsupportedSearchType() throws IOException { + RestRequest restRequest = new FakeRestRequest(); + SearchRequest searchRequest = createSearchRequest(); + IntConsumer setSize = mock(IntConsumer.class); + restRequest.params().put("search_type", "query_and_fetch"); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> RestSearchAction.parseSearchRequest(searchRequest, restRequest, null, namedWriteableRegistry, setSize) + ); + assertEquals("Unsupported search type [query_and_fetch]", exception.getMessage()); + } + public void testEqualsAndHashcode() throws IOException { checkEqualsAndHashCode(createSearchRequest(), SearchRequest::new, this::mutate); } @@ -268,10 +287,7 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add( () -> mutation.searchType( - randomValueOtherThan( - searchRequest.searchType(), - () -> randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH) - ) + randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)) ) ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 9697f4cee0d58..da8ccc9e121e0 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -421,6 +421,27 @@ public void testDerivedFieldsParsingAndSerializationObjectType() throws IOExcept } } + public void testSearchPipelineParsingAndSerialization() throws IOException { + String restContent = "{ \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10, \"search_pipeline\": \"my_pipeline\" }"; + String expectedContent = "{\"from\":0,\"size\":10,\"query\":{\"match_all\":{\"boost\":1.0}},\"search_pipeline\":\"my_pipeline\"}"; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser); + searchSourceBuilder = rewrite(searchSourceBuilder); + + try (BytesStreamOutput output = new BytesStreamOutput()) { + searchSourceBuilder.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { + SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); + String actualContent = deserializedBuilder.toString(); + assertEquals(expectedContent, actualContent); + assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode()); + assertNotSame(searchSourceBuilder, deserializedBuilder); + } + } + } + } + public void testAggsParsing() throws IOException { { String restContent = "{\n" diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index f5857922fdff2..b52205996f34b 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -969,6 +969,64 @@ public void testInlinePipeline() throws Exception { } } + public void testInlineDefinedPipeline() throws Exception { + SearchPipelineService searchPipelineService = createWithProcessors(); + + SearchPipelineMetadata metadata = new SearchPipelineMetadata( + Map.of( + "p1", + new PipelineConfiguration( + "p1", + new BytesArray( + "{" + + "\"request_processors\": [{ \"scale_request_size\": { \"scale\" : 2 } }]," + + "\"response_processors\": [{ \"fixed_score\": { \"score\" : 2 } }]" + + "}" + ), + MediaTypeRegistry.JSON + ) + ) + + ); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousState = clusterState; + clusterState = ClusterState.builder(clusterState) + .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata)) + .build(); + searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState)); + + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).pipeline("p1"); + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + searchRequest.pipeline(searchRequest.source().pipeline()); + + // Verify pipeline + PipelinedRequest pipelinedRequest = syncTransformRequest( + searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver) + ); + Pipeline pipeline = pipelinedRequest.getPipeline(); + assertEquals("p1", pipeline.getId()); + assertEquals(1, pipeline.getSearchRequestProcessors().size()); + assertEquals(1, pipeline.getSearchResponseProcessors().size()); + + // Verify that pipeline transforms request + assertEquals(200, pipelinedRequest.source().size()); + + int size = 10; + SearchHit[] hits = new SearchHit[size]; + for (int i = 0; i < size; i++) { + hits[i] = new SearchHit(i, "doc" + i, Collections.emptyMap(), Collections.emptyMap()); + hits[i].score(i); + } + SearchHits searchHits = new SearchHits(hits, new TotalHits(size * 2, TotalHits.Relation.EQUAL_TO), size); + SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 10, null, null); + + SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse); + for (int i = 0; i < size; i++) { + assertEquals(2.0, transformedResponse.getHits().getHits()[i].getScore(), 0.0001); + } + } + public void testInfo() { SearchPipelineService searchPipelineService = createWithProcessors(); SearchPipelineInfo info = searchPipelineService.info(); From ae22e3ff32ef15a6af302c50872f1fa0e8e140fe Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 25 Sep 2024 22:09:00 -0400 Subject: [PATCH 146/357] Update 390_search_as_you_type.yml (#16065) Signed-off-by: Andriy Redko --- .../rest-api-spec/test/search/390_search_as_you_type.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml index 689b3edfd1066..f29fa3f9aa987 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/390_search_as_you_type.yml @@ -30,8 +30,8 @@ teardown: --- "Test search_as_you_type data type supports multi-fields": - skip: - version: " - 2.99.99" - reason: "the bug was fixed since 3.0.0" + version: " - 2.17.99" + reason: "the bug was fixed since 2.18.0" - do: indices.get_mapping: { From 58888672e648193baaa6bc88ed3730207c3522ff Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:02:43 +0530 Subject: [PATCH 147/357] Allow RemotePublication configured nodes to join remote cluster in mixed mode (#15994) Signed-off-by: Rajiv Kumar Vaidyanathan --- .../RemotePublicationConfigurationIT.java | 273 ++++++++++++++++++ .../MigrationBaseTestCase.java | 4 +- .../coordination/JoinTaskExecutor.java | 15 +- 3 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java new file mode 100644 index 0000000000000..1d19a4bfd1af5 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -0,0 +1,273 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotemigration.MigrationBaseTestCase; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.junit.Before; + +import java.util.Collection; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * Tests the compatibility between types of nodes based on the configured repositories + * Non Remote node [No Repositories configured] + * Remote Publish Configured Node [Cluster State + Routing Table] + * Remote Node [Cluster State + Segment + Translog] + * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemotePublicationConfigurationIT extends MigrationBaseTestCase { + private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; + + @Override + protected Collection> nodePlugins() { + /* Adding the following mock plugins: + - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync + - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated + */ + return Stream.concat( + super.nodePlugins().stream(), + Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) + ).collect(Collectors.toList()); + } + + @Before + public void setUp() throws Exception { + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } + super.setUp(); + } + + public Settings.Builder remotePublishConfiguredNodeSetting() { + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String routingTableRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + ROUTING_TABLE_REPO_NAME + ); + String routingTableRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + ROUTING_TABLE_REPO_NAME + ); + + Settings.Builder builder = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME) + .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); + return builder; + } + + public Settings.Builder remoteWithRoutingTableNodeSetting() { + // Remote Cluster with Routing table + return Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + REPOSITORY_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); + } + + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ensureStableCluster(6); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { + // Remote Cluster without Routing table + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // Remote Repo with Routing table + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + internalCluster().startClusterManagerOnlyNode(); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + + Settings settings = Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + internalCluster().startClusterManagerOnlyNode(settings); + + ensureStableCluster(4); + ensureGreen(); + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java index 17a9c3ddbe317..a82e6d45ce0f6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java @@ -57,6 +57,8 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected static final String ROUTING_TABLE_REPO_NAME = "remote-routing-repo"; + protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2"; protected Path segmentRepoPath; @@ -72,7 +74,7 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase { randomAlphaOfLength(5) ); - void setAddRemote(boolean addRemote) { + public void setAddRemote(boolean addRemote) { this.addRemote = addRemote; } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 838b5723b217b..13033b670d44b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -420,9 +420,14 @@ public static void ensureIndexCompatibility(final Version nodeVersion, Metadata * ensures that the joining node has a version that's compatible with all current nodes */ public static void ensureNodesCompatibility(final DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { - final Version minNodeVersion = currentNodes.getMinNodeVersion(); - final Version maxNodeVersion = currentNodes.getMaxNodeVersion(); - ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion); + try { + final Version minNodeVersion = currentNodes.getMinNodeVersion(); + final Version maxNodeVersion = currentNodes.getMaxNodeVersion(); + ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion); + } catch (Exception e) { + logger.error("Exception in NodesCompatibility validation", e); + throw e; + } } /** @@ -539,9 +544,11 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); List reposToSkip = new ArrayList<>(1); + // find a remote node which has routing table configured Optional remoteRoutingTableNode = existingNodes.stream() .filter( - node -> node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null + node -> node.isRemoteStoreNode() + && node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null ) .findFirst(); // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node. From 949b31f986c559d90017bef5d5d36455de75a77a Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 26 Sep 2024 11:47:20 -0400 Subject: [PATCH 148/357] Ignore google-cloud-storage and google-api-client major version upgrade for dependabot (#16072) * Ignore google-cloud-storage major version upgrade for dependabot Signed-off-by: Craig Perkins * Include google-api-client Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c4f4d59ea1fc..af83f8b2edac2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -722,6 +722,12 @@ updates: - directory: /plugins/ open-pull-requests-limit: 1 package-ecosystem: gradle + ignore: + # For all packages, ignore all major versions to minimize breaking issues + - dependency-name: "com.google.cloud:google-cloud-storage" + update-types: [ "version-update:semver-major" ] + - dependency-name: "com.google.api-client:google-api-client" + update-types: [ "version-update:semver-major" ] schedule: interval: weekly labels: From 007d1b97a010d2b7ec15f257c4c9c65d61a789c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:54:51 -0400 Subject: [PATCH 149/357] Bump com.nimbusds:nimbus-jose-jwt from 9.40 to 9.41.1 in /plugins/repository-azure (#16038) * Bump com.nimbusds:nimbus-jose-jwt in /plugins/repository-azure Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.40 to 9.41.1. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.41.1..9.40) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 | 1 - .../repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 create mode 100644 plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 935901d410178..e88f85453109e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 616da0be62c30..2892bdba51ba6 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -63,7 +63,7 @@ dependencies { api "net.java.dev.jna:jna-platform:${versions.jna}" api 'com.microsoft.azure:msal4j:1.17.1' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' - api 'com.nimbusds:nimbus-jose-jwt:9.40' + api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api 'com.nimbusds:content-type:2.3' api 'com.nimbusds:lang-tag:1.7' // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart, diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 deleted file mode 100644 index 83228caf233cc..0000000000000 --- a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42b1dfa0360e4062951b070bac52dd8d96fd7b38 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 new file mode 100644 index 0000000000000..71fa950cb9530 --- /dev/null +++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 @@ -0,0 +1 @@ +35532a88e1b49a623ec97fd276cc208ea525b6bc \ No newline at end of file From 7caca26fcb2dfee9e28ad09cc2de0e5d0248dba9 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Thu, 26 Sep 2024 16:09:07 -0700 Subject: [PATCH 150/357] Updates version to fix BWC for SearchSourceBuilder (#16086) Signed-off-by: Owais --- .../org/opensearch/search/builder/SearchSourceBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index dd4e4d073cb1b..9c438401b9fbe 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -299,7 +299,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException { derivedFields = in.readList(DerivedField::new); } } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { searchPipeline = in.readOptionalString(); } } @@ -382,7 +382,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeList(derivedFields); } } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalString(searchPipeline); } } From e8b02c9545d7908ae1a3ea2d109c06ff3e0c07a5 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:02:24 +0530 Subject: [PATCH 151/357] Shallow snapshot v2 - create snapshot validations in a cluster state update (#15939) --------- Signed-off-by: Gaurav Bafna --- .../remotestore/RemoteRestoreSnapshotIT.java | 126 +++-- .../remotestore/RemoteSnapshotIT.java | 89 ++++ .../snapshots/CloneSnapshotV2IT.java | 106 ++-- .../snapshots/ConcurrentSnapshotsV2IT.java | 486 ++++++++++++++++++ .../cluster/SnapshotsInProgress.java | 141 ++++- .../snapshots/SnapshotsService.java | 410 ++++++++++----- ...SnapshotsInProgressSerializationTests.java | 6 +- 7 files changed, 1134 insertions(+), 230 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 0acb578e2e7bf..a0183e89bfce2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -11,6 +11,8 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -25,6 +27,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.Index; @@ -43,14 +46,11 @@ import org.opensearch.repositories.RepositoryData; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotRestoreException; import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.nio.file.Files; @@ -63,6 +63,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,48 +80,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { - private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; - private Path remoteRepoPath; - - @Before - public void setup() { - remoteRepoPath = randomRepoPath().toAbsolutePath(); - } - - @After - public void teardown() { - clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) - .build(); - } - - private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { - Settings.Builder settingsBuilder = Settings.builder() - .put(super.indexSettings()) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); - return settingsBuilder; - } - - private void indexDocuments(Client client, String indexName, int numOfDocs) { - indexDocuments(client, indexName, 0, numOfDocs); - } - - private void indexDocuments(Client client, String indexName, int fromId, int toId) { - for (int i = fromId; i < toId; i++) { - String id = Integer.toString(i); - client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); - } - client.admin().indices().prepareFlush(indexName).get(); - } +public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT { private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) { for (int i = 0; i < numOfDocs; i++) { @@ -997,6 +957,75 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); } + public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + Thread thread = new Thread(() -> { + try { + String snapshotName = "snapshot-earlier-master"; + internalCluster().nonClusterManagerClient() + .admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName) + .setWaitForCompletion(true) + .setMasterNodeTimeout(TimeValue.timeValueSeconds(60)) + .get(); + + } catch (Exception ignored) {} + }); + thread.start(); + + // stop existing master + final String clusterManagerNode = internalCluster().getClusterManagerName(); + stopNode(clusterManagerNode); + + // Validate that we have greater one snapshot has been created + String snapshotName = "new-snapshot"; + try { + client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, snapshotName).setWaitForCompletion(true).get(); + } catch (Exception e) { + logger.info("Exception while creating new-snapshot", e); + } + + // Validate that snapshot is present in repository data + assertBusy(() -> { + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1)); + }, 30, TimeUnit.SECONDS); + thread.join(); + } + public void testCreateSnapshotV2WithRedIndex() throws Exception { internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); @@ -1315,11 +1344,4 @@ public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception { createV1SnapshotThread.join(); } - private Settings pinnedTimestampSettings() { - Settings settings = Settings.builder() - .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) - .build(); - return settings; - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java new file mode 100644 index 0000000000000..836871b8251d1 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.concurrent.ExecutionException; + +import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING; + +public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase { + protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; + protected Path remoteRepoPath; + + @Before + public void setup() { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } + + @After + public void teardown() { + clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) + .build(); + } + + protected Settings pinnedTimestampSettings() { + Settings settings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .build(); + return settings; + } + + protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + return settingsBuilder; + } + + protected void indexDocuments(Client client, String indexName, int numOfDocs) { + indexDocuments(client, indexName, 0, numOfDocs); + } + + void indexDocuments(Client client, String indexName, int fromId, int toId) { + for (int i = fromId; i < toId; i++) { + String id = Integer.toString(i); + client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get(); + } + client.admin().indices().prepareFlush(indexName).get(); + } + + protected void setFailRate(String repoName, int value) throws ExecutionException, InterruptedException { + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName }); + GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get(); + RepositoryMetadata rmd = res.repositories().get(0); + Settings.Builder settings = Settings.builder() + .put("location", rmd.settings().get("location")) + .put(REPOSITORIES_FAILRATE_SETTING.getKey(), value); + createRepository(repoName, ReloadableFsRepository.TYPE, settings); + } + +} diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java index c6744ae62db60..69e85b13548e0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java @@ -34,11 +34,15 @@ import org.opensearch.action.ActionRunnable; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.rest.RestStatus; @@ -50,8 +54,10 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.nio.file.Path; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -134,27 +140,32 @@ public void testCloneShallowCopyV2() throws Exception { assertTrue(response.isAcknowledged()); awaitClusterManagerFinishRepoOperations(); + AtomicReference cloneSnapshotId = new AtomicReference<>(); // Validate that snapshot is present in repository data - PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); - repository.getRepositoryData(repositoryDataPlainActionFutureClone); - - repositoryData = repositoryDataPlainActionFutureClone.get(); - assertEquals(repositoryData.getSnapshotIds().size(), 2); - boolean foundCloneInRepoData = false; - SnapshotId cloneSnapshotId = null; - for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { - if (snapshotId.getName().equals("test_clone_snapshot1")) { - foundCloneInRepoData = true; - cloneSnapshotId = snapshotId; + waitUntil(() -> { + PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFutureClone); + + RepositoryData repositoryData1; + try { + repositoryData1 = repositoryDataPlainActionFutureClone.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); } - } - final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; + for (SnapshotId snapshotId : repositoryData1.getSnapshotIds()) { + if (snapshotId.getName().equals("test_clone_snapshot1")) { + cloneSnapshotId.set(snapshotId); + return true; + } + } + return false; + }, 90, TimeUnit.SECONDS); + + final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId.get(); SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) ); - assertTrue(foundCloneInRepoData); - assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); for (String index : sourceSnapshotInfo.indices()) { assertTrue(cloneSnapshotInfo.indices().contains(index)); @@ -259,19 +270,21 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception { assertThat(sourceSnapshotInfoV1.state(), equalTo(SnapshotState.SUCCESS)); assertThat(sourceSnapshotInfoV1.successfulShards(), greaterThan(0)); assertThat(sourceSnapshotInfoV1.successfulShards(), equalTo(sourceSnapshotInfoV1.totalShards())); - assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L)); + // assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L)); + AtomicReference repositoryDataAtomicReference = new AtomicReference<>(); + awaitClusterManagerFinishRepoOperations(); // Validate that snapshot is present in repository data - PlainActionFuture repositoryDataV1PlainActionFuture = new PlainActionFuture<>(); - BlobStoreRepository repositoryV1 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( - RepositoriesService.class - ).repository(snapshotRepoName); - repositoryV1.getRepositoryData(repositoryDataV1PlainActionFuture); - - repositoryData = repositoryDataV1PlainActionFuture.get(); + assertBusy(() -> { + Metadata metadata = clusterAdmin().prepareState().get().getState().metadata(); + RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE); + assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).generation()); + assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration()); - assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfoV1.snapshotId())); - assertEquals(repositoryData.getSnapshotIds().size(), 2); + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response = client().admin().cluster().getSnapshots(request).actionGet(); + assertEquals(2, response.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); // clone should get created for v2 snapshot AcknowledgedResponse response = client().admin() @@ -289,31 +302,28 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception { ).repository(snapshotRepoName); repositoryCloneV2.getRepositoryData(repositoryDataCloneV2PlainActionFuture); - repositoryData = repositoryDataCloneV2PlainActionFuture.get(); - - assertEquals(repositoryData.getSnapshotIds().size(), 3); - boolean foundCloneInRepoData = false; - SnapshotId cloneSnapshotId = null; - for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { - if (snapshotId.getName().equals(cloneSnapshotV2)) { - foundCloneInRepoData = true; - cloneSnapshotId = snapshotId; - } - } - final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId; - SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( - f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) - ); + // Validate that snapshot is present in repository data + assertBusy(() -> { + Metadata metadata = clusterAdmin().prepareState().get().getState().metadata(); + RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE); + assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).generation()); + assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration()); + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + assertEquals(3, response2.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); - assertTrue(foundCloneInRepoData); // pinned timestamp value in clone snapshot v2 matches source snapshot v2 - assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); - for (String index : sourceSnapshotInfo.indices()) { - assertTrue(cloneSnapshotInfo.indices().contains(index)); - + GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName, new String[] { sourceSnapshotV2, cloneSnapshotV2 }); + GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); + + SnapshotInfo sourceInfo = response2.getSnapshots().get(0); + SnapshotInfo cloneInfo = response2.getSnapshots().get(1); + assertEquals(sourceInfo.getPinnedTimestamp(), cloneInfo.getPinnedTimestamp()); + assertEquals(sourceInfo.totalShards(), cloneInfo.totalShards()); + for (String index : sourceInfo.indices()) { + assertTrue(cloneInfo.indices().contains(index)); } - assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); - } public void testRestoreFromClone() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java new file mode 100644 index 0000000000000..f20fddb6af26c --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -0,0 +1,486 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.Client; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.remotestore.RemoteSnapshotIT; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class ConcurrentSnapshotsV2IT extends RemoteSnapshotIT { + + public void testLongRunningSnapshotDontAllowConcurrentSnapshot() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + + try { + String snapshotName = "snapshot-concurrent"; + client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).get(); + fail(); + } catch (Exception e) {} + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testCreateSnapshotFailInFinalize() throws Exception { + final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + unblockNode(repoName, clusterManagerNode); + expectThrows(SnapshotException.class, snapshotFuture::actionGet); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testCreateSnapshotV2MasterSwitch() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String clusterManagerNode = internalCluster().getClusterManagerName(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + + // Fail the cluster manager + stopNode(clusterManagerNode); + + ensureGreen(); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + + } + + public void testPinnedTimestampFailSnapshot() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + // fail segment repo - this is to fail the timestamp pinning + setFailRate(BASE_REMOTE_REPO, 100); + + try { + String snapshotName = "snapshot-fail"; + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + fail(); + } catch (Exception e) {} + + setFailRate(BASE_REMOTE_REPO, 0); + String snapshotName = "snapshot-success"; + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + } + + public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String snapshotRepoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + int concurrentSnapshots = 5; + + // Prepare threads for concurrent snapshot creation + List threads = new ArrayList<>(); + + for (int i = 0; i < concurrentSnapshots; i++) { + int snapshotIndex = i; + Thread thread = new Thread(() -> { + try { + String snapshotName = "snapshot-concurrent-" + snapshotIndex; + CreateSnapshotResponse createSnapshotResponse2 = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName)); + assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); + } catch (Exception e) {} + }); + threads.add(thread); + } + // start all threads + for (Thread thread : threads) { + thread.start(); + } + + // Wait for all threads to complete + for (Thread thread : threads) { + thread.join(); + } + + // Validate that only one snapshot has been created + Repository repository = internalCluster().getInstance(RepositoriesService.class).repository(snapshotRepoName); + PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFuture); + + RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); + assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); + } + + public void testLongRunningSnapshotDontAllowConcurrentClone() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + + final CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet(); + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone"); + awaitNumberOfSnapshotsInProgress(1); + + final ActionFuture snapshotFuture2 = startCloneSnapshot(repoName, sourceSnap, "snapshot-clone-2"); + assertThrows(ConcurrentSnapshotExecutionException.class, snapshotFuture2::actionGet); + + unblockNode(repoName, clusterManagerName); + assertThrows(SnapshotException.class, snapshotFuture2::actionGet); + + snapshotFuture.get(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + } + + public void testCloneSnapshotFailInFinalize() throws Exception { + final String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + CreateSnapshotResponse sourceResp = startFullSnapshot(repoName, sourceSnap).actionGet(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + unblockNode(repoName, clusterManagerNode); + assertThrows(SnapshotException.class, snapshotFuture::actionGet); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo is cleaned up + CreateSnapshotResponse csr = snapshotFuture2.actionGet(); + + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), sourceResp.getSnapshotInfo())); + } + + public void testCloneSnapshotV2MasterSwitch() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + String sourceSnap = "snapshot-source"; + CreateSnapshotResponse csr = startFullSnapshot(repoName, sourceSnap).actionGet(); + + String clusterManagerNode = internalCluster().getClusterManagerName(); + + blockClusterManagerFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotFuture = startCloneSnapshot(repoName, sourceSnap, "snapshot-queued"); + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(30L)); + + // Fail the cluster manager + stopNode(clusterManagerNode); + + ensureGreen(); + + final ActionFuture snapshotFuture2 = startFullSnapshot(repoName, "snapshot-success"); + // Second create works out cleanly since the repo + CreateSnapshotResponse csr2 = snapshotFuture2.actionGet(); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(2, snapInfo.size()); + assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); + } + + protected ActionFuture startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) { + logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName); + return clusterAdmin().prepareCloneSnapshot(repoName, sourceSnapshotName, snapshotName).setIndices("*").execute(); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index d658f38430dd9..595f6d54fd9be 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -134,6 +134,38 @@ public static Entry startedEntry( ); } + public static Entry startedEntry( + Snapshot snapshot, + boolean includeGlobalState, + boolean partial, + List indices, + List dataStreams, + long startTime, + long repositoryStateId, + final Map shards, + Map userMetadata, + Version version, + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 + ) { + return new SnapshotsInProgress.Entry( + snapshot, + includeGlobalState, + partial, + completed(shards.values()) ? State.SUCCESS : State.STARTED, + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + null, + userMetadata, + version, + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 + ); + } + /** * Creates the initial snapshot clone entry * @@ -168,8 +200,39 @@ public static Entry startClone( version, source, Map.of(), - false // initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with - // clone jobs + false, + false// initialising to false, will be updated in startCloning method of SnapshotsService while updating entry with + // clone jobs + ); + } + + public static Entry startClone( + Snapshot snapshot, + SnapshotId source, + List indices, + long startTime, + long repositoryStateId, + Version version, + boolean remoteStoreIndexShallowCopyV2 + ) { + return new SnapshotsInProgress.Entry( + snapshot, + true, + false, + State.STARTED, + indices, + Collections.emptyList(), + startTime, + repositoryStateId, + Map.of(), + null, + Collections.emptyMap(), + version, + source, + Map.of(), + remoteStoreIndexShallowCopyV2, + remoteStoreIndexShallowCopyV2// initialising to false, will be updated in startCloning method of SnapshotsService + // while updating entry with clone jobs ); } @@ -183,6 +246,8 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation private final Snapshot snapshot; private final boolean includeGlobalState; private final boolean remoteStoreIndexShallowCopy; + + private final boolean remoteStoreIndexShallowCopyV2; private final boolean partial; /** * Map of {@link ShardId} to {@link ShardSnapshotStatus} tracking the state of each shard snapshot operation. @@ -212,6 +277,42 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation @Nullable private final String failure; + public Entry( + Snapshot snapshot, + boolean includeGlobalState, + boolean partial, + State state, + List indices, + List dataStreams, + long startTime, + long repositoryStateId, + final Map shards, + String failure, + Map userMetadata, + Version version, + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 + ) { + this( + snapshot, + includeGlobalState, + partial, + state, + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + failure, + userMetadata, + version, + null, + Map.of(), + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 + ); + } + // visible for testing, use #startedEntry and copy constructors in production code public Entry( Snapshot snapshot, @@ -243,7 +344,8 @@ public Entry( version, null, Map.of(), - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + false ); } @@ -262,7 +364,8 @@ private Entry( Version version, @Nullable SnapshotId source, @Nullable final Map clones, - boolean remoteStoreIndexShallowCopy + boolean remoteStoreIndexShallowCopy, + boolean remoteStoreIndexShallowCopyV2 ) { this.state = state; this.snapshot = snapshot; @@ -284,7 +387,9 @@ private Entry( this.clones = Collections.unmodifiableMap(clones); } this.remoteStoreIndexShallowCopy = remoteStoreIndexShallowCopy; - assert assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones); + this.remoteStoreIndexShallowCopyV2 = remoteStoreIndexShallowCopyV2; + assert this.remoteStoreIndexShallowCopyV2 + || assertShardsConsistent(this.source, this.state, this.indices, this.shards, this.clones); } private Entry(StreamInput in) throws IOException { @@ -307,6 +412,11 @@ private Entry(StreamInput in) throws IOException { } else { remoteStoreIndexShallowCopy = false; } + if (in.getVersion().onOrAfter(Version.CURRENT)) { + remoteStoreIndexShallowCopyV2 = in.readBoolean(); + } else { + remoteStoreIndexShallowCopyV2 = false; + } } private static boolean assertShardsConsistent( @@ -428,7 +538,8 @@ public Entry withRepoGen(long newRepoGen) { version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -451,7 +562,8 @@ public Entry withClones(final Map update version, source, updatedClones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -471,7 +583,8 @@ public Entry withRemoteStoreIndexShallowCopy(final boolean remoteStoreIndexShall version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -527,7 +640,8 @@ public Entry fail(final Map shards, State state, S version, source, clones, - remoteStoreIndexShallowCopy + remoteStoreIndexShallowCopy, + remoteStoreIndexShallowCopyV2 ); } @@ -614,6 +728,10 @@ public boolean remoteStoreIndexShallowCopy() { return remoteStoreIndexShallowCopy; } + public boolean remoteStoreIndexShallowCopyV2() { + return remoteStoreIndexShallowCopyV2; + } + public Map userMetadata() { return userMetadata; } @@ -678,6 +796,7 @@ public boolean equals(Object o) { if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; if (remoteStoreIndexShallowCopy != entry.remoteStoreIndexShallowCopy) return false; + if (remoteStoreIndexShallowCopyV2 != entry.remoteStoreIndexShallowCopyV2) return false; return true; } @@ -695,6 +814,7 @@ public int hashCode() { result = 31 * result + (source == null ? 0 : source.hashCode()); result = 31 * result + clones.hashCode(); result = 31 * result + (remoteStoreIndexShallowCopy ? 1 : 0); + result = 31 * result + (remoteStoreIndexShallowCopyV2 ? 1 : 0); return result; } @@ -766,6 +886,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeBoolean(remoteStoreIndexShallowCopy); } + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeBoolean(remoteStoreIndexShallowCopyV2); + } } @Override diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 23f6deff3715d..22b2a72b36026 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -468,26 +468,33 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi long pinnedTimestamp = System.currentTimeMillis(); final String repositoryName = request.repository(); final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot()); - validate(repositoryName, snapshotName); - final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot Repository repository = repositoriesService.repository(repositoryName); + validate(repositoryName, snapshotName); + repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) { + private SnapshotsInProgress.Entry newEntry; - if (repository.isReadOnly()) { - listener.onFailure( - new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository") - ); - return; - } + private SnapshotId snapshotId; - final Snapshot snapshot = new Snapshot(repositoryName, snapshotId); - ClusterState currentState = clusterService.state(); - final Map userMeta = repository.adaptUserMetadata(request.userMetadata()); - try { - final StepListener repositoryDataListener = new StepListener<>(); - repositoriesService.getRepositoryData(repositoryName, repositoryDataListener); + private Snapshot snapshot; + + boolean enteredLoop; + + @Override + public ClusterState execute(ClusterState currentState) { + // move to in progress + snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot + Repository repository = repositoriesService.repository(repositoryName); + + if (repository.isReadOnly()) { + listener.onFailure( + new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository") + ); + } + + snapshot = new Snapshot(repositoryName, snapshotId); + final Map userMeta = repository.adaptUserMetadata(request.userMetadata()); - repositoryDataListener.whenComplete(repositoryData -> { createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); List indices = new ArrayList<>(currentState.metadata().indices().keySet()); @@ -498,7 +505,7 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi request.indices() ); - logger.trace("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices); + logger.info("[{}][{}] creating snapshot-v2 for indices [{}]", repositoryName, snapshotName, indices); final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final List runningSnapshots = snapshots.entries(); @@ -509,74 +516,139 @@ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionLi IndexId.DEFAULT_SHARD_PATH_TYPE ); final Version version = minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null); - final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData( - currentState.metadata(), - currentState.routingTable(), - indexIds, - repositoryData - ); if (repositoryData.getGenId() == RepositoryData.UNKNOWN_REPO_GEN) { logger.debug("[{}] was aborted before starting", snapshot); throw new SnapshotException(snapshot, "Aborted on initialization"); } + + Map shards = new HashMap<>(); + + newEntry = SnapshotsInProgress.startedEntry( + new Snapshot(repositoryName, snapshotId), + request.includeGlobalState(), + request.partial(), + indexIds, + dataStreams, + threadPool.absoluteTimeInMillis(), + repositoryData.getGenId(), + shards, + userMeta, + version, + true, + true + ); + final List newEntries = new ArrayList<>(runningSnapshots); + newEntries.add(newEntry); + + // Entering finalize loop here to prevent concurrent snapshots v2 snapshots + enteredLoop = tryEnterRepoLoop(repositoryName); + if (enteredLoop == false) { + throw new ConcurrentSnapshotExecutionException( + repositoryName, + snapshotName, + "cannot start snapshot-v2 while a repository is in finalization state" + ); + } + return ClusterState.builder(currentState) + .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(new ArrayList<>(newEntries))) + .build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to create snapshot-v2", repositoryName, snapshotName), e); + listener.onFailure(e); + if (enteredLoop) { + leaveRepoLoop(repositoryName); + } + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) { + final ShardGenerations shardGenerations = buildShardsGenerationFromRepositoryData( + newState.metadata(), + newState.routingTable(), + newEntry.indices(), + repositoryData + ); + final List dataStreams = indexNameExpressionResolver.dataStreamNames( + newState, + request.indicesOptions(), + request.indices() + ); final SnapshotInfo snapshotInfo = new SnapshotInfo( - snapshot.getSnapshotId(), + snapshotId, shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), - dataStreams, + newEntry.dataStreams(), pinnedTimestamp, null, System.currentTimeMillis(), shardGenerations.totalShards(), Collections.emptyList(), request.includeGlobalState(), - userMeta, + newEntry.userMetadata(), true, pinnedTimestamp ); - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager"); - } + final Version version = minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null); final StepListener pinnedTimestampListener = new StepListener<>(); - pinnedTimestampListener.whenComplete(repoData -> { listener.onResponse(snapshotInfo); }, listener::onFailure); - repository.finalizeSnapshot( - shardGenerations, - repositoryData.getGenId(), - metadataForSnapshot(currentState.metadata(), request.includeGlobalState(), false, dataStreams, indexIds), - snapshotInfo, - version, - state -> state, - Priority.IMMEDIATE, - new ActionListener() { - @Override - public void onResponse(RepositoryData repositoryData) { - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - failSnapshotCompletionListeners( - snapshot, - new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") - ); - listener.onFailure( - new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager") - ); - return; + pinnedTimestampListener.whenComplete(repoData -> { + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot(newState.metadata(), request.includeGlobalState(), false, dataStreams, newEntry.indices()), + snapshotInfo, + version, + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + leaveRepoLoop(repositoryName); + if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException( + repositoryName, + snapshotName, + "Aborting snapshot-v2, no longer cluster manager" + ) + ); + return; + } + listener.onResponse(snapshotInfo); } - updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener); - } - @Override - public void onFailure(Exception e) { - logger.error("Failed to upload files to snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error("Failed to finalize snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); + leaveRepoLoop(repositoryName); + // cleaning up in progress snapshot here + stateWithoutSnapshotV2(newState); + listener.onFailure(e); + } } - } - ); + ); + }, e -> { + logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} {} ", repositoryName, snapshotName, e); + leaveRepoLoop(repositoryName); + // cleaning up in progress snapshot here + stateWithoutSnapshotV2(newState); + listener.onFailure(e); + }); + updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener); + } - }, listener::onFailure); - } catch (Exception e) { - assert false : new AssertionError(e); - logger.error("Snapshot-v2 {} creation failed with exception {}", snapshot.getSnapshotId().getName(), e); - listener.onFailure(e); - } + @Override + public TimeValue timeout() { + return request.clusterManagerNodeTimeout(); + } + + }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } private void createSnapshotPreValidations( @@ -770,12 +842,24 @@ public void cloneSnapshotV2( private SnapshotId sourceSnapshotId; private List indicesForSnapshot; + boolean enteredRepoLoop; + @Override public ClusterState execute(ClusterState currentState) { createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final List runningSnapshots = snapshots.entries(); + // Entering finalize loop here to prevent concurrent snapshots v2 snapshots + enteredRepoLoop = tryEnterRepoLoop(repositoryName); + if (enteredRepoLoop == false) { + throw new ConcurrentSnapshotExecutionException( + repositoryName, + snapshotName, + "cannot start snapshot-v2 while a repository is in finalization state" + ); + } + sourceSnapshotId = repositoryData.getSnapshotIds() .stream() .filter(src -> src.getName().equals(request.source())) @@ -799,14 +883,14 @@ public ClusterState execute(ClusterState currentState) { indicesForSnapshot.add(indexId.getName()); } } - newEntry = SnapshotsInProgress.startClone( snapshot, sourceSnapshotId, repositoryData.resolveIndices(indicesForSnapshot), threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), - minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null) + minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null), + true ); final List newEntries = new ArrayList<>(runningSnapshots); newEntries.add(newEntry); @@ -817,6 +901,9 @@ public ClusterState execute(ClusterState currentState) { public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to clone snapshot-v2", repositoryName, snapshotName), e); listener.onFailure(e); + if (enteredRepoLoop) { + leaveRepoLoop(repositoryName); + } } @Override @@ -843,67 +930,80 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl true, snapshotInfo.getPinnedTimestamp() ); - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2 clone, no longer cluster manager"); } final StepListener pinnedTimestampListener = new StepListener<>(); pinnedTimestampListener.whenComplete(repoData -> { - logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); - listener.onResponse(null); - }, listener::onFailure); - repository.finalizeSnapshot( - shardGenerations, - repositoryData.getGenId(), - metadataForSnapshot( - currentState.metadata(), - newEntry.includeGlobalState(), - false, - newEntry.dataStreams(), - newEntry.indices() - ), - cloneSnapshotInfo, - repositoryData.getVersion(sourceSnapshotId), - state -> stateWithoutSnapshot(state, snapshot), - Priority.IMMEDIATE, - new ActionListener() { - @Override - public void onResponse(RepositoryData repositoryData) { - if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - failSnapshotCompletionListeners( - snapshot, - new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") - ); - listener.onFailure( - new SnapshotException( - repositoryName, - snapshotName, - "Aborting Snapshot-v2 clone, no longer cluster manager" - ) - ); - return; + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot( + currentState.metadata(), + newEntry.includeGlobalState(), + false, + newEntry.dataStreams(), + newEntry.indices() + ), + cloneSnapshotInfo, + repositoryData.getVersion(sourceSnapshotId), + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + leaveRepoLoop(repositoryName); + if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException( + repositoryName, + snapshotName, + "Aborting Snapshot-v2 clone, no longer cluster manager" + ) + ); + return; + } + logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); + listener.onResponse(null); } - cloneSnapshotPinnedTimestamp( - repositoryData, - sourceSnapshotId, - snapshot, - snapshotInfo.getPinnedTimestamp(), - pinnedTimestampListener - ); - } - @Override - public void onFailure(Exception e) { - logger.error( - "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ", - repositoryName, - snapshotName - ); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error( + "Failed to upload files to snapshot repo {} for clone snapshot-v2 {} ", + repositoryName, + snapshotName + ); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + } } - } + ); + }, e -> { + logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} ", repositoryName, snapshotName); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + }); + + cloneSnapshotPinnedTimestamp( + repositoryData, + sourceSnapshotId, + snapshot, + snapshotInfo.getPinnedTimestamp(), + pinnedTimestampListener ); - - }, listener::onFailure); + }, e -> { + logger.error("Failed to retrieve snapshot info for snapshot-v2 {} {} ", repositoryName, snapshotName); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + }); } @Override @@ -1486,6 +1586,13 @@ public void applyClusterState(ClusterChangedEvent event) { // cluster-manager SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final boolean newClusterManager = event.previousState().nodes().isLocalNodeElectedClusterManager() == false; + + if (newClusterManager && snapshotsInProgress.entries().isEmpty() == false) { + // clean up snapshot v2 in progress or clone v2 present. + // Snapshot v2 create and clone are sync operation . In case of cluster manager failures in midst , we won't + // send ack to caller and won't continue on new cluster manager . Caller will need to retry it. + stateWithoutSnapshotV2(event.state()); + } processExternalChanges( newClusterManager || removedNodesCleanupNeeded(snapshotsInProgress, event.nodesDelta().removedNodes()), event.routingTableChanged() && waitingShardsStartedOrUnassigned(snapshotsInProgress, event) @@ -1597,7 +1704,14 @@ private void processExternalChanges(boolean changedNodes, boolean startShards) { @Override public ClusterState execute(ClusterState currentState) { RoutingTable routingTable = currentState.routingTable(); - final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + // Removing shallow snapshots v2 as we we take care of these in stateWithoutSnapshotV2() + snapshots = SnapshotsInProgress.of( + snapshots.entries() + .stream() + .filter(snapshot -> snapshot.remoteStoreIndexShallowCopyV2() == false) + .collect(Collectors.toList()) + ); DiscoveryNodes nodes = currentState.nodes(); boolean changed = false; final EnumSet statesToUpdate; @@ -1654,7 +1768,7 @@ public ClusterState execute(ClusterState currentState) { changed = true; logger.debug("[{}] was found in dangling INIT or ABORTED state", snapshot); } else { - if (snapshot.state().completed() || completed(snapshot.shards().values())) { + if ((snapshot.state().completed() || completed(snapshot.shards().values()))) { finishedSnapshots.add(snapshot); } updatedSnapshotEntries.add(snapshot); @@ -2180,6 +2294,59 @@ private static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sn return readyDeletions(result).v1(); } + private void stateWithoutSnapshotV2(ClusterState state) { + SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + boolean changed = false; + ArrayList entries = new ArrayList<>(); + for (SnapshotsInProgress.Entry entry : snapshots.entries()) { + if (entry.remoteStoreIndexShallowCopyV2()) { + changed = true; + } else { + entries.add(entry); + } + } + if (changed) { + logger.info("Cleaning up in progress v2 snapshots now"); + clusterService.submitStateUpdateTask( + "remove in progress snapshot v2 after cluster manager switch", + new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + boolean changed = false; + ArrayList entries = new ArrayList<>(); + for (SnapshotsInProgress.Entry entry : snapshots.entries()) { + if (entry.remoteStoreIndexShallowCopyV2()) { + changed = true; + } else { + entries.add(entry); + } + } + if (changed) { + return ClusterState.builder(currentState) + .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(entries))) + .build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + // execute never fails , so we should never hit this. + logger.warn( + () -> new ParameterizedMessage( + "failed to remove in progress snapshot v2 state after cluster manager switch {}", + e + ), + e + ); + } + } + ); + } + } + /** * Removes record of running snapshot from cluster state and notifies the listener when this action is complete. This method is only * used when the snapshot fails for some reason. During normal operation the snapshot repository will remove the @@ -3330,6 +3497,9 @@ public boolean assertAllListenersResolved() { + " on [" + localNode + "]"; + if (repositoryOperations.isEmpty() == false) { + logger.info("Not empty"); + } assert repositoryOperations.isEmpty() : "Found leaked snapshots to finalize " + repositoryOperations + " on [" + localNode + "]"; return true; } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index 8fd1f44286094..d79cb62b6b7ac 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -214,7 +214,11 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { assert (curr_entry.remoteStoreIndexShallowCopy() == false); } } - try (StreamInput in = out.bytes().streamInput()) { + + BytesStreamOutput out2 = new BytesStreamOutput(); + out2.setVersion(Version.V_2_9_0); + snapshotsInProgress.writeTo(out2); + try (StreamInput in = out2.bytes().streamInput()) { in.setVersion(Version.V_2_9_0); actualSnapshotsInProgress = new SnapshotsInProgress(in); assert in.available() == 0; From 8d19cb9bc177d068e12fd7d5ac060da91c6bbe33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:53:05 -0400 Subject: [PATCH 152/357] Bump org.roaringbitmap:RoaringBitmap from 1.2.1 to 1.3.0 in /server (#16040) * Bump org.roaringbitmap:RoaringBitmap from 1.2.1 to 1.3.0 in /server Bumps [org.roaringbitmap:RoaringBitmap](https://github.com/RoaringBitmap/RoaringBitmap) from 1.2.1 to 1.3.0. - [Release notes](https://github.com/RoaringBitmap/RoaringBitmap/releases) - [Commits](https://github.com/RoaringBitmap/RoaringBitmap/compare/1.2.1...1.3.0) --- updated-dependencies: - dependency-name: org.roaringbitmap:RoaringBitmap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + server/build.gradle | 2 +- server/licenses/RoaringBitmap-1.2.1.jar.sha1 | 1 - server/licenses/RoaringBitmap-1.3.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 server/licenses/RoaringBitmap-1.2.1.jar.sha1 create mode 100644 server/licenses/RoaringBitmap-1.3.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e88f85453109e..798802f11ebad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) diff --git a/server/build.gradle b/server/build.gradle index 0cc42ad690eab..83a04ef12d13b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -127,7 +127,7 @@ dependencies { api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap - implementation 'org.roaringbitmap:RoaringBitmap:1.2.1' + implementation 'org.roaringbitmap:RoaringBitmap:1.3.0' testImplementation(project(":test:framework")) { // tests use the locally compiled version of server diff --git a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 b/server/licenses/RoaringBitmap-1.2.1.jar.sha1 deleted file mode 100644 index ef8cd48c7a388..0000000000000 --- a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -828eb489b5e8c8762f2471010e9c7f20c7de596d \ No newline at end of file diff --git a/server/licenses/RoaringBitmap-1.3.0.jar.sha1 b/server/licenses/RoaringBitmap-1.3.0.jar.sha1 new file mode 100644 index 0000000000000..2e48289c4f7af --- /dev/null +++ b/server/licenses/RoaringBitmap-1.3.0.jar.sha1 @@ -0,0 +1 @@ +a46ce7a2dc494da69700ab421f081b1583857f6d \ No newline at end of file From 0363a64ac988a7bfb96a651e810d3b74cef65c2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:03:38 -0400 Subject: [PATCH 153/357] Bump com.nimbusds:nimbus-jose-jwt in /test/fixtures/hdfs-fixture (#16095) Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.40 to 9.41.1. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.41.1..9.40) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 21fe28bfd835d..99b0386a48808 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -79,7 +79,7 @@ dependencies { api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.26.3' api 'org.apache.commons:commons-configuration2:2.11.0' - api 'com.nimbusds:nimbus-jose-jwt:9.40' + api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api ('org.apache.kerby:kerb-admin:2.1.0') { exclude group: "org.jboss.xnio" exclude group: "org.jline" From 78860ace6cf7512c3e83a82bf116fc03ddb9e35d Mon Sep 17 00:00:00 2001 From: Sarat Vemulapalli Date: Fri, 27 Sep 2024 16:52:52 -0700 Subject: [PATCH 154/357] Registering disable user-agent setting with OpenSearch (#16052) * Registering allowlist setting in user-agent Signed-off-by: Sarat Vemulapalli * Adding unit tests Signed-off-by: Sarat Vemulapalli --------- Signed-off-by: Sarat Vemulapalli --- .../ingest/geoip/IngestGeoIpModulePluginTests.java | 7 +++++++ .../ingest/useragent/IngestUserAgentModulePlugin.java | 3 ++- .../ingest/useragent/IngestUserAgentModulePluginTests.java | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java index 9446ec1228532..ac6b87aefb3ec 100644 --- a/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java +++ b/modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java @@ -35,6 +35,7 @@ import com.maxmind.geoip2.model.AbstractResponse; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.env.TestEnvironment; import org.opensearch.ingest.Processor; @@ -126,6 +127,12 @@ public void testAllowListNotSpecified() throws IOException { } } + public void testSettingsRegistration() { + final IngestGeoIpModulePlugin plugin = new IngestGeoIpModulePlugin(); + final List> settings = plugin.getSettings(); + assertTrue(settings.contains(IngestGeoIpModulePlugin.PROCESSORS_ALLOWLIST_SETTING)); + } + private void runAllowListTest(List allowList) throws IOException { Settings.Builder settingsBuilder = Settings.builder(); createDb(settingsBuilder); diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java index bac90d20b44e1..f5e2e34278880 100644 --- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java @@ -44,6 +44,7 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -152,6 +153,6 @@ static Map createUserAgentParsers(Path userAgentConfigD @Override public List> getSettings() { - return Collections.singletonList(CACHE_SIZE_SETTING); + return Arrays.asList(CACHE_SIZE_SETTING, PROCESSORS_ALLOWLIST_SETTING); } } diff --git a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java index 31fdafff1188a..563158026c37f 100644 --- a/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java +++ b/modules/ingest-user-agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java @@ -8,6 +8,7 @@ package org.opensearch.ingest.useragent; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.env.TestEnvironment; import org.opensearch.ingest.Processor; @@ -89,6 +90,12 @@ public void testAllowListNotSpecified() throws IOException { } } + public void testSettingsRegistration() { + final IngestUserAgentModulePlugin plugin = new IngestUserAgentModulePlugin(); + final List> settings = plugin.getSettings(); + assertTrue(settings.contains(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING)); + } + private void runAllowListTest(List allowList) throws IOException { final Settings settings = settingsBuilder.putList(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), allowList) .build(); From b50117b0649a05792d5abc72da70991989726657 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Sep 2024 08:58:20 -0400 Subject: [PATCH 155/357] Bump com.maxmind.geoip2:geoip2 from 4.2.0 to 4.2.1 in /modules/ingest-geoip (#16042) * Bump com.maxmind.geoip2:geoip2 in /modules/ingest-geoip Bumps [com.maxmind.geoip2:geoip2](https://github.com/maxmind/GeoIP2-java) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/maxmind/GeoIP2-java/releases) - [Changelog](https://github.com/maxmind/GeoIP2-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/maxmind/GeoIP2-java/compare/v4.2.0...v4.2.1) --- updated-dependencies: - dependency-name: com.maxmind.geoip2:geoip2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + modules/ingest-geoip/build.gradle | 2 +- modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 | 1 - modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 798802f11ebad..e35ca9c2fcb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) +- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index c0ff155ce1038..162ac6f9ad620 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -39,7 +39,7 @@ opensearchplugin { } dependencies { - api('com.maxmind.geoip2:geoip2:4.2.0') + api('com.maxmind.geoip2:geoip2:4.2.1') // geoip2 dependencies: api('com.maxmind.db:maxmind-db:3.1.0') api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 deleted file mode 100644 index b6bfeeb9da60b..0000000000000 --- a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78ff932dc13ac41dd1f0fd9e7405a7f4ad815ce0 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 new file mode 100644 index 0000000000000..a9dc5483ac727 --- /dev/null +++ b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 @@ -0,0 +1 @@ +9dbf8a8bea88a33e88c46eb3f503721b4bd08b90 \ No newline at end of file From 1563e1af2d8ac2e4fdf555e4f81e389560cedc05 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Sat, 28 Sep 2024 18:44:35 +0530 Subject: [PATCH 156/357] Fix for race condition in node-join/node-left loop (#15521) * Add custom connect to node for handleJoinRequest Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + .../cluster/coordination/NodeJoinLeftIT.java | 355 ++++++++++++++++++ .../cluster/NodeConnectionsService.java | 21 +- .../cluster/coordination/Coordinator.java | 23 +- .../cluster/coordination/Publication.java | 2 +- .../PublicationTransportHandler.java | 2 +- .../service/ClusterApplierService.java | 4 + .../org/opensearch/discovery/Discovery.java | 5 + .../main/java/org/opensearch/node/Node.java | 1 + .../transport/ClusterConnectionManager.java | 30 ++ .../transport/ConnectionManager.java | 4 + .../transport/RemoteConnectionManager.java | 10 + .../transport/TransportService.java | 12 + .../cluster/NodeConnectionsServiceTests.java | 191 ++++++++++ .../snapshots/SnapshotResiliencyTests.java | 28 +- .../org/opensearch/test/NoopDiscovery.java | 6 + .../ClusterConnectionManagerTests.java | 44 +++ .../AbstractCoordinatorTestCase.java | 32 +- .../coordination/DeterministicTaskQueue.java | 11 + .../org/opensearch/test/TestLogsAppender.java | 74 ++++ .../transport/StubbableConnectionManager.java | 10 + 21 files changed, 844 insertions(+), 22 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java create mode 100644 test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e35ca9c2fcb64..07637eaae3306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) +- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java new file mode 100644 index 0000000000000..014e2bf642a4d --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/NodeJoinLeftIT.java @@ -0,0 +1,355 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.cluster.coordination; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.cluster.NodeConnectionsService; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.MockEngineFactoryPlugin; +import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.plugins.Plugin; +import org.opensearch.tasks.Task; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.TestLogsAppender; +import org.opensearch.test.store.MockFSIndexStore; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.test.transport.StubbableTransport; +import org.opensearch.transport.ClusterConnectionManager; +import org.opensearch.transport.TransportChannel; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; +import org.opensearch.transport.TransportService; +import org.junit.After; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME; +import static org.hamcrest.Matchers.is; + +/** + Check https://github.com/opensearch-project/OpenSearch/issues/4874 and + https://github.com/opensearch-project/OpenSearch/pull/15521 for context + */ +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +public class NodeJoinLeftIT extends OpenSearchIntegTestCase { + + private TestLogsAppender testLogsAppender; + private String clusterManager; + private String redNodeName; + private LoggerContext loggerContext; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + MockTransportService.TestPlugin.class, + MockFSIndexStore.TestPlugin.class, + InternalSettingsPlugin.class, + MockEngineFactoryPlugin.class + ); + } + + @Override + protected void beforeIndexDeletion() throws Exception { + super.beforeIndexDeletion(); + internalCluster().assertConsistentHistoryBetweenTranslogAndLuceneIndex(); + internalCluster().assertSeqNos(); + internalCluster().assertSameDocIdsOnShards(); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + // Add any other specific messages you want to capture + List messagesToCapture = Arrays.asList("failed to join", "IllegalStateException"); + testLogsAppender = new TestLogsAppender(messagesToCapture); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerContext.updateLoggers(); + + String indexName = "test"; + final Settings nodeSettings = Settings.builder() + .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), "100ms") + .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "10s") + .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "200ms") + .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "100ms") + .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) + .put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "100ms") + .build(); + // start a 3 node cluster with 1 cluster-manager + this.clusterManager = internalCluster().startNode(nodeSettings); + internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build()); + this.redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build()); + + // validate the 3 node cluster is up + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes(">=3").get(); + assertThat(response.isTimedOut(), is(false)); + + // create an index + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings( + Settings.builder() + .put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .get(); + } + + @After + public void tearDown() throws Exception { + testLogsAppender.clearCapturedLogs(); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerContext.updateLoggers(); + super.tearDown(); + } + + public void testClusterStabilityWhenJoinRequestHappensDuringNodeLeftTask() throws Exception { + + ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager); + // Simulate a slow applier on the cm to delay node-left state application + clusterManagerClsService.addStateApplier(event -> { + if (event.nodesRemoved()) { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + // Toggle to succeed/fail the followerchecker to simulate the initial node leaving. + AtomicBoolean succeedFollowerChecker = new AtomicBoolean(); + + // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false + FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> { + if (succeedFollowerChecker.get()) { + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition"); + }); + MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + redNodeName + ); + redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour); + + // Loop runs 5 times to ensure race condition gets reproduced + testLogsAppender.clearCapturedLogs(); + for (int i = 0; i < 5; i++) { + logger.info("--> simulating followerchecker failure to trigger node-left"); + succeedFollowerChecker.set(false); + ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response1.isTimedOut(), is(false)); + + // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins + logger.info("--> re-enabling normal followerchecker and validating cluster is stable"); + succeedFollowerChecker.set(true); + ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response2.isTimedOut(), is(false)); + + Thread.sleep(1000); + // checking again to validate stability and ensure node did not leave + ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response3.isTimedOut(), is(false)); + } + + succeedFollowerChecker.set(true); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + + // assert that join requests fail with the right exception + boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS) + && testLogsAppender.waitForLog( + "IllegalStateException[cannot make a new connection as disconnect to node", + 30, + TimeUnit.SECONDS + ); + assertTrue("Expected log was not found within the timeout period", logFound); + } + + public void testClusterStabilityWhenDisconnectDuringSlowNodeLeftTask() throws Exception { + ClusterService clusterManagerClsService = internalCluster().getInstance(ClusterService.class, clusterManager); + // Simulate a slow applier on the cm to delay node-left state application + clusterManagerClsService.addStateApplier(event -> { + if (event.nodesRemoved()) { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + // Toggle to succeed/fail the followerchecker to simulate the initial node leaving. + AtomicBoolean succeedFollowerChecker = new AtomicBoolean(); + + // Simulate followerchecker failure on 1 node when succeedFollowerChecker is false + FollowerCheckerBehaviour simulatedFailureBehaviour = new FollowerCheckerBehaviour(() -> { + if (succeedFollowerChecker.get()) { + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + throw new NodeHealthCheckFailureException("fake followerchecker failure simulated by test to repro race condition"); + }); + MockTransportService cmTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + clusterManager + ); + MockTransportService redTransportService = (MockTransportService) internalCluster().getInstance( + TransportService.class, + redNodeName + ); + redTransportService.addRequestHandlingBehavior(FOLLOWER_CHECK_ACTION_NAME, simulatedFailureBehaviour); + + // Loop runs 5 times to ensure race condition gets reproduced + testLogsAppender.clearCapturedLogs(); + for (int i = 0; i < 5; i++) { + // Fail followerchecker by force to trigger node disconnect and node left + logger.info("--> simulating followerchecker failure to trigger node-left"); + succeedFollowerChecker.set(false); + Thread.sleep(1000); + + // Trigger a node disconnect while node-left task is still processing + logger.info( + "--> triggering a simulated disconnect on red node, after the follower checker failed to see how node-left task deals with this" + ); + cmTransportService.disconnectFromNode(redTransportService.getLocalDiscoNode()); + + ClusterHealthResponse response1 = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response1.isTimedOut(), is(false)); + + // once we know a node has left, we can re-enable followerchecker to work normally and validate node rejoins + logger.info("--> re-enabling normal followerchecker and validating cluster is stable"); + succeedFollowerChecker.set(true); + ClusterHealthResponse response2 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response2.isTimedOut(), is(false)); + + Thread.sleep(1000); + // checking again to validate stability and ensure node did not leave + ClusterHealthResponse response3 = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response3.isTimedOut(), is(false)); + } + + succeedFollowerChecker.set(true); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + + // assert that join requests fail with the right exception + boolean logFound = testLogsAppender.waitForLog("failed to join", 30, TimeUnit.SECONDS); + assertTrue("Expected log was not found within the timeout period", logFound); + logFound = testLogsAppender.waitForLog( + "IllegalStateException[cannot make a new connection as disconnect to node", + 30, + TimeUnit.SECONDS + ); + assertTrue("Expected log was not found within the timeout period", logFound); + } + + public void testRestartDataNode() throws Exception { + + Settings redNodeDataPathSettings = internalCluster().dataPathSettings(redNodeName); + logger.info("-> stopping data node"); + internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(redNodeName)); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response.isTimedOut(), is(false)); + + logger.info("-> restarting stopped node"); + internalCluster().startNode(Settings.builder().put("node.name", redNodeName).put(redNodeDataPathSettings).build()); + response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + } + + public void testRestartCmNode() throws Exception { + + Settings cmNodeSettings = internalCluster().dataPathSettings(clusterManager); + + logger.info("-> stopping cluster-manager node"); + internalCluster().stopRandomNode(settings -> settings.get("node.name").equals(clusterManager)); + ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes("2").get(); + assertThat(response.isTimedOut(), is(false)); + + logger.info("-> restarting stopped node"); + internalCluster().startNode(Settings.builder().put("node.name", clusterManager).put(cmNodeSettings).build()); + response = client().admin().cluster().prepareHealth().setWaitForNodes("3").get(); + assertThat(response.isTimedOut(), is(false)); + } + + private class FollowerCheckerBehaviour implements StubbableTransport.RequestHandlingBehavior { + private final Runnable connectionBreaker; + + private FollowerCheckerBehaviour(Runnable connectionBreaker) { + this.connectionBreaker = connectionBreaker; + } + + @Override + public void messageReceived( + TransportRequestHandler handler, + TransportRequest request, + TransportChannel channel, + Task task + ) throws Exception { + + connectionBreaker.run(); + handler.messageReceived(request, channel, task); + } + } +} diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java index 1c12c260b3929..8ce11c8183cf6 100644 --- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java @@ -103,10 +103,10 @@ public class NodeConnectionsService extends AbstractLifecycleComponent { // contains an entry for every node in the latest cluster state, as well as for nodes from which we are in the process of // disconnecting - private final Map targetsByNode = new HashMap<>(); + protected final Map targetsByNode = new HashMap<>(); private final TimeValue reconnectInterval; - private volatile ConnectionChecker connectionChecker; + protected volatile ConnectionChecker connectionChecker; @Inject public NodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) { @@ -115,6 +115,11 @@ public NodeConnectionsService(Settings settings, ThreadPool threadPool, Transpor this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings); } + // exposed for testing + protected ConnectionTarget createConnectionTarget(DiscoveryNode discoveryNode) { + return new ConnectionTarget(discoveryNode); + } + /** * Connect to all the given nodes, but do not disconnect from any extra nodes. Calls the completion handler on completion of all * connection attempts to _new_ nodes, but not on attempts to re-establish connections to nodes that are already known. @@ -159,6 +164,14 @@ public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) runnables.forEach(Runnable::run); } + public void setPendingDisconnections(Set nodes) { + nodes.forEach(transportService::setPendingDisconnection); + } + + public void clearPendingDisconnections() { + transportService.clearPendingDisconnections(); + } + /** * Disconnect from any nodes to which we are currently connected which do not appear in the given nodes. Does not wait for the * disconnections to complete, because they might have to wait for ongoing connection attempts first. @@ -211,7 +224,7 @@ private void awaitPendingActivity(Runnable onCompletion) { * nodes which are in the process of disconnecting. The onCompletion handler is called after all ongoing connection/disconnection * attempts have completed. */ - private void connectDisconnectedTargets(Runnable onCompletion) { + protected void connectDisconnectedTargets(Runnable onCompletion) { final List runnables = new ArrayList<>(); synchronized (mutex) { final Collection connectionTargets = targetsByNode.values(); @@ -321,7 +334,7 @@ private enum ActivityType { * * @opensearch.internal */ - private class ConnectionTarget { + protected class ConnectionTarget { private final DiscoveryNode discoveryNode; private PlainListenableActionFuture future = PlainListenableActionFuture.newListenableFuture(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 13a57d93f03f0..9859abe503eaa 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -42,6 +42,7 @@ import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.LocalClusterUpdateTask; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.ClusterFormationFailureHelper.ClusterFormationState; import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; @@ -187,6 +188,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; + private NodeConnectionsService nodeConnectionsService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -418,7 +420,11 @@ PublishWithJoinResponse handlePublishRequest(PublishRequest publishRequest) { synchronized (mutex) { final DiscoveryNode sourceNode = publishRequest.getAcceptedState().nodes().getClusterManagerNode(); - logger.trace("handlePublishRequest: handling [{}] from [{}]", publishRequest, sourceNode); + logger.debug( + "handlePublishRequest: handling version [{}] from [{}]", + publishRequest.getAcceptedState().getVersion(), + sourceNode + ); if (sourceNode.equals(getLocalNode()) && mode != Mode.LEADER) { // Rare case in which we stood down as leader between starting this publication and receiving it ourselves. The publication @@ -630,7 +636,6 @@ private void handleJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback transportService.connectToNode(joinRequest.getSourceNode(), ActionListener.wrap(ignore -> { final ClusterState stateForJoinValidation = getStateForClusterManagerService(); - if (stateForJoinValidation.nodes().isLocalNodeElectedClusterManager()) { onJoinValidators.forEach(a -> a.accept(joinRequest.getSourceNode(), stateForJoinValidation)); if (stateForJoinValidation.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) { @@ -814,6 +819,10 @@ public void onFailure(String source, Exception e) { public ClusterTasksResult execute(ClusterState currentState) { if (currentState.nodes().isLocalNodeElectedClusterManager() == false) { allocationService.cleanCaches(); + // This set only needs to be maintained on active cluster-manager + // This is cleaned up to avoid stale entries which would block future reconnections + logger.trace("Removing all pending disconnections as part of cluster-manager cleanup"); + nodeConnectionsService.clearPendingDisconnections(); } return unchanged(); } @@ -914,11 +923,18 @@ public DiscoveryStats stats() { @Override public void startInitialJoin() { synchronized (mutex) { + logger.trace("Starting initial join, becoming candidate"); becomeCandidate("startInitialJoin"); } clusterBootstrapService.scheduleUnconfiguredBootstrap(); } + @Override + public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) { + assert this.nodeConnectionsService == null : "nodeConnectionsService is already set"; + this.nodeConnectionsService = nodeConnectionsService; + } + @Override protected void doStop() { configuredHostsResolver.stop(); @@ -1356,6 +1372,9 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) currentPublication = Optional.of(publication); final DiscoveryNodes publishNodes = publishRequest.getAcceptedState().nodes(); + // marking pending disconnects before publish + // if a nodes tries to send a joinRequest while it is pending disconnect, it should fail + nodeConnectionsService.setPendingDisconnections(new HashSet<>(clusterChangedEvent.nodesDelta().removedNodes())); leaderChecker.setCurrentNodes(publishNodes); followersChecker.setCurrentNodes(publishNodes); lagDetector.setTrackedNodes(publishNodes); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java index 43801a05dbc24..3f7218939be92 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Publication.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Publication.java @@ -85,7 +85,7 @@ public Publication(PublishRequest publishRequest, AckListener ackListener, LongS } public void start(Set faultyNodes) { - logger.trace("publishing {} to {}", publishRequest, publicationTargets); + logger.debug("publishing version {} to {}", publishRequest.getAcceptedState().getVersion(), publicationTargets); for (final DiscoveryNode faultyNode : faultyNodes) { onFaultyNode(faultyNode); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index cdf331b7bb577..caed2b6eceb49 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -542,7 +542,7 @@ public String executor() { } public void sendClusterState(DiscoveryNode destination, ActionListener listener) { - logger.debug("sending cluster state over transport to node: {}", destination.getName()); + logger.trace("sending cluster state over transport to node: {}", destination.getName()); if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); sendFullClusterState(destination, listener); diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index 47080cfbde692..d0b6f812e9ee2 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -502,6 +502,7 @@ private void runTask(UpdateTask task) { try { applyChanges(task, previousClusterState, newClusterState, stopWatch); TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS)); + // At this point, cluster state appliers and listeners are completed logger.debug( "processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})", task.source, @@ -510,6 +511,7 @@ private void runTask(UpdateTask task) { newClusterState.stateUUID() ); warnAboutSlowTaskIfNeeded(executionTime, task.source, stopWatch); + // Then we call the ClusterApplyListener of the task task.listener.onSuccess(task.source); } catch (Exception e) { TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, currentTimeInMillis() - startTimeMS)); @@ -578,6 +580,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl logger.debug("apply cluster state with version {}", newClusterState.version()); callClusterStateAppliers(clusterChangedEvent, stopWatch); + logger.debug("completed calling appliers of cluster state for version {}", newClusterState.version()); nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes()); @@ -594,6 +597,7 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl state.set(newClusterState); callClusterStateListeners(clusterChangedEvent, stopWatch); + logger.debug("completed calling listeners of cluster state for version {}", newClusterState.version()); } protected void connectToNodesAndWait(ClusterState newClusterState) { diff --git a/server/src/main/java/org/opensearch/discovery/Discovery.java b/server/src/main/java/org/opensearch/discovery/Discovery.java index 9d6807b6522c9..6d9fb1f4985df 100644 --- a/server/src/main/java/org/opensearch/discovery/Discovery.java +++ b/server/src/main/java/org/opensearch/discovery/Discovery.java @@ -32,6 +32,7 @@ package org.opensearch.discovery; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.cluster.coordination.ClusterStatePublisher; import org.opensearch.common.lifecycle.LifecycleComponent; @@ -54,4 +55,8 @@ public interface Discovery extends LifecycleComponent, ClusterStatePublisher { */ void startInitialJoin(); + /** + * Sets the NodeConnectionsService which is an abstraction used for connection management + */ + void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index a8d4ebcf23dab..4962d72d8728a 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1602,6 +1602,7 @@ public Node start() throws NodeValidationException { injector.getInstance(GatewayService.class).start(); Discovery discovery = injector.getInstance(Discovery.class); + discovery.setNodeConnectionsService(nodeConnectionsService); clusterService.getClusterManagerService().setClusterStatePublisher(discovery::publish); // Start the transport service now so the publish address will be added to the local disco node in ClusterService diff --git a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java index e634323d58269..3a3e8c964b6c5 100644 --- a/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/ClusterConnectionManager.java @@ -64,6 +64,15 @@ public class ClusterConnectionManager implements ConnectionManager { private final ConcurrentMap connectedNodes = ConcurrentCollections.newConcurrentMap(); private final ConcurrentMap> pendingConnections = ConcurrentCollections.newConcurrentMap(); + /** + This set is used only by cluster-manager nodes. + Nodes are marked as pending disconnect right before cluster state publish phase. + They are cleared up as part of cluster state apply commit phase + This is to avoid connections from being made to nodes that are in the process of leaving the cluster + Note: If a disconnect is initiated while a connect is in progress, this Set will not handle this case. + Callers need to ensure that connects and disconnects are sequenced. + */ + private final Set pendingDisconnections = ConcurrentCollections.newConcurrentSet(); private final AbstractRefCounted connectingRefCounter = new AbstractRefCounted("connection manager") { @Override protected void closeInternal() { @@ -122,12 +131,19 @@ public void connectToNode( ConnectionValidator connectionValidator, ActionListener listener ) throws ConnectTransportException { + logger.trace("connecting to node [{}]", node); ConnectionProfile resolvedProfile = ConnectionProfile.resolveConnectionProfile(connectionProfile, defaultProfile); if (node == null) { listener.onFailure(new ConnectTransportException(null, "can't connect to a null node")); return; } + // if node-left is still in progress, we fail the connect request early + if (pendingDisconnections.contains(node)) { + listener.onFailure(new IllegalStateException("cannot make a new connection as disconnect to node [" + node + "] is pending")); + return; + } + if (connectingRefCounter.tryIncRef() == false) { listener.onFailure(new IllegalStateException("connection manager is closed")); return; @@ -170,6 +186,7 @@ public void connectToNode( conn.addCloseListener(ActionListener.wrap(() -> { logger.trace("unregistering {} after connection close and marking as disconnected", node); connectedNodes.remove(node, finalConnection); + pendingDisconnections.remove(node); connectionListener.onNodeDisconnected(node, conn); })); } @@ -226,6 +243,19 @@ public void disconnectFromNode(DiscoveryNode node) { // if we found it and removed it we close nodeChannels.close(); } + pendingDisconnections.remove(node); + logger.trace("Removed node [{}] from pending disconnections list", node); + } + + @Override + public void setPendingDisconnection(DiscoveryNode node) { + logger.trace("marking disconnection as pending for node: [{}]", node); + pendingDisconnections.add(node); + } + + @Override + public void clearPendingDisconnections() { + pendingDisconnections.clear(); } /** diff --git a/server/src/main/java/org/opensearch/transport/ConnectionManager.java b/server/src/main/java/org/opensearch/transport/ConnectionManager.java index 10cfc2907098f..ebd5ccf29c8cc 100644 --- a/server/src/main/java/org/opensearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/ConnectionManager.java @@ -65,6 +65,10 @@ void connectToNode( void disconnectFromNode(DiscoveryNode node); + void setPendingDisconnection(DiscoveryNode node); + + void clearPendingDisconnections(); + Set getAllConnectedNodes(); int size(); diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java index bd646f10df517..52f29bea8050d 100644 --- a/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java +++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionManager.java @@ -114,6 +114,16 @@ public void disconnectFromNode(DiscoveryNode node) { delegate.disconnectFromNode(node); } + @Override + public void setPendingDisconnection(DiscoveryNode node) { + delegate.setPendingDisconnection(node); + } + + @Override + public void clearPendingDisconnections() { + delegate.clearPendingDisconnections(); + } + @Override public ConnectionProfile getConnectionProfile() { return delegate.getConnectionProfile(); diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index fff6d82b23c7e..fe8631aa5ca3d 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -773,6 +773,18 @@ public void disconnectFromNode(DiscoveryNode node) { connectionManager.disconnectFromNode(node); } + public void setPendingDisconnection(DiscoveryNode node) { + connectionManager.setPendingDisconnection(node); + } + + /** + * Wipes out all pending disconnections. + * This is called on cluster-manager failover to remove stale entries + */ + public void clearPendingDisconnections() { + connectionManager.clearPendingDisconnections(); + } + public void addMessageListener(TransportMessageListener listener) { messageListener.listeners.add(listener); } diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java index 4cf82f1dabab3..4500860c937ea 100644 --- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java @@ -35,6 +35,9 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.opensearch.OpenSearchTimeoutException; import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; @@ -53,9 +56,11 @@ import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.TestLogsAppender; import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.ClusterConnectionManager; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.ConnectionProfile; import org.opensearch.transport.Transport; @@ -69,6 +74,7 @@ import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -77,6 +83,7 @@ import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import static java.util.Collections.emptySet; @@ -86,12 +93,15 @@ import static org.opensearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; import static org.opensearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; public class NodeConnectionsServiceTests extends OpenSearchTestCase { private ThreadPool threadPool; private TransportService transportService; private Map> nodeConnectionBlocks; + private TestLogsAppender testLogsAppender; + private LoggerContext loggerContext; private List generateNodes() { List nodes = new ArrayList<>(); @@ -490,6 +500,108 @@ public void testDebugLogging() throws IllegalAccessException { } } + public void testConnectionCheckerRetriesIfPendingDisconnection() throws InterruptedException { + final Settings.Builder settings = Settings.builder(); + final long reconnectIntervalMillis = 50; + settings.put(CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), reconnectIntervalMillis + "ms"); + + final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue( + builder().put(NODE_NAME_SETTING.getKey(), "node").build(), + random() + ); + + MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool()); + TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool()); + transportService.start(); + transportService.acceptIncomingRequests(); + + final TestNodeConnectionsService service = new TestNodeConnectionsService( + settings.build(), + deterministicTaskQueue.getThreadPool(), + transportService + ); + service.start(); + + // setup the connections + final DiscoveryNode node = new DiscoveryNode("node0", buildNewFakeTransportAddress(), Version.CURRENT); + + final DiscoveryNodes nodes = DiscoveryNodes.builder().add(node).build(); + + final AtomicBoolean connectionCompleted = new AtomicBoolean(); + service.connectToNodes(nodes, () -> connectionCompleted.set(true)); + deterministicTaskQueue.runAllRunnableTasks(); + assertTrue(connectionCompleted.get()); + + // reset any logs as we want to assert for exceptions that show up after this + // reset connect to node count to assert for later + logger.info("--> resetting captured logs and counters"); + testLogsAppender.clearCapturedLogs(); + // this ensures we only track connection attempts that happen after the disconnection + transportService.resetConnectToNodeCallCount(); + + // block connection checker reconnection attempts until after we set pending disconnections + logger.info("--> disabling connection checker, and triggering disconnect"); + service.setShouldReconnect(false); + transportService.disconnectFromNode(node); + + // set pending disconnections to true to fail future reconnection attempts + final long maxDisconnectionTime = 1000; + deterministicTaskQueue.scheduleNow(new Runnable() { + @Override + public void run() { + logger.info("--> setting pending disconnections to fail next connection attempts"); + service.setPendingDisconnections(new HashSet<>(Collections.singleton(node))); + } + + @Override + public String toString() { + return "scheduled disconnection of " + node; + } + }); + // our task queue will have the first task as the runnable to set pending disconnections + // here we re-enable the connection checker to enqueue next tasks for attempting reconnection + logger.info("--> re-enabling reconnection checker"); + service.setShouldReconnect(true); + + final long maxReconnectionTime = 2000; + final int expectedReconnectionAttempts = 10; + + // this will first run the task to set the pending disconnections, then will execute the reconnection tasks + // exit early when we have enough reconnection attempts + logger.info("--> running tasks in order until expected reconnection attempts"); + runTasksInOrderUntilExpectedReconnectionAttempts( + deterministicTaskQueue, + maxDisconnectionTime + maxReconnectionTime, + transportService, + expectedReconnectionAttempts + ); + logger.info("--> verifying that connectionchecker tried to reconnect"); + + // assert that the connections failed + assertFalse("connected to " + node, transportService.nodeConnected(node)); + + // assert that we saw at least the required number of reconnection attempts, and the exceptions that showed up are as expected + logger.info("--> number of reconnection attempts: {}", transportService.getConnectToNodeCallCount()); + assertThat( + "Did not see enough reconnection attempts from connection checker", + transportService.getConnectToNodeCallCount(), + greaterThan(expectedReconnectionAttempts) + ); + boolean logFound = testLogsAppender.waitForLog("failed to connect", 1, TimeUnit.SECONDS) + && testLogsAppender.waitForLog( + "IllegalStateException: cannot make a new connection as disconnect to node", + 1, + TimeUnit.SECONDS + ); + assertTrue("Expected log for reconnection failure was not found in the required time period", logFound); + + // clear the pending disconnections and ensure the connection gets re-established automatically by connectionchecker + logger.info("--> clearing pending disconnections to allow connections to re-establish"); + service.clearPendingDisconnections(); + runTasksUntil(deterministicTaskQueue, maxDisconnectionTime + maxReconnectionTime + 2 * reconnectIntervalMillis); + assertConnectedExactlyToNodes(transportService, nodes); + } + private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long endTimeMillis) { while (deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) { if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) { @@ -501,6 +613,24 @@ private void runTasksUntil(DeterministicTaskQueue deterministicTaskQueue, long e deterministicTaskQueue.runAllRunnableTasks(); } + private void runTasksInOrderUntilExpectedReconnectionAttempts( + DeterministicTaskQueue deterministicTaskQueue, + long endTimeMillis, + TestTransportService transportService, + int expectedReconnectionAttempts + ) { + // break the loop if we timeout or if we have enough reconnection attempts + while ((deterministicTaskQueue.getCurrentTimeMillis() < endTimeMillis) + && (transportService.getConnectToNodeCallCount() <= expectedReconnectionAttempts)) { + if (deterministicTaskQueue.hasRunnableTasks() && randomBoolean()) { + deterministicTaskQueue.runNextTask(); + } else if (deterministicTaskQueue.hasDeferredTasks()) { + deterministicTaskQueue.advanceTime(); + } + } + deterministicTaskQueue.runAllRunnableTasksInEnqueuedOrder(); + } + private void ensureConnections(NodeConnectionsService service) { final PlainActionFuture future = new PlainActionFuture<>(); service.ensureConnections(() -> future.onResponse(null)); @@ -526,6 +656,16 @@ private void assertConnected(TransportService transportService, Iterable messagesToCapture = Arrays.asList("failed to connect", "IllegalStateException"); + testLogsAppender = new TestLogsAppender(messagesToCapture); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.addAppender(testLogsAppender, null, null); + loggerContext.updateLoggers(); ThreadPool threadPool = new TestThreadPool(getClass().getName()); this.threadPool = threadPool; nodeConnectionBlocks = newConcurrentMap(); @@ -537,6 +677,14 @@ public void setUp() throws Exception { @Override @After public void tearDown() throws Exception { + testLogsAppender.clearCapturedLogs(); + loggerContext = (LoggerContext) LogManager.getContext(false); + Configuration config = loggerContext.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(NodeConnectionsService.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerConfig = config.getLoggerConfig(ClusterConnectionManager.class.getName()); + loggerConfig.removeAppender(testLogsAppender.getName()); + loggerContext.updateLoggers(); transportService.stop(); ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); threadPool = null; @@ -545,6 +693,8 @@ public void tearDown() throws Exception { private final class TestTransportService extends TransportService { + private final AtomicInteger connectToNodeCallCount = new AtomicInteger(0); + private TestTransportService(Transport transport, ThreadPool threadPool) { super( Settings.EMPTY, @@ -588,6 +738,47 @@ public void connectToNode(DiscoveryNode node, ActionListener listener) thr } else { super.connectToNode(node, listener); } + logger.info("calling connectToNode"); + connectToNodeCallCount.incrementAndGet(); + } + + public int getConnectToNodeCallCount() { + return connectToNodeCallCount.get(); + } + + public void resetConnectToNodeCallCount() { + connectToNodeCallCount.set(0); + } + } + + private class TestNodeConnectionsService extends NodeConnectionsService { + private boolean shouldReconnect = true; + + public TestNodeConnectionsService(Settings settings, ThreadPool threadPool, TransportService transportService) { + super(settings, threadPool, transportService); + } + + public void setShouldReconnect(boolean shouldReconnect) { + this.shouldReconnect = shouldReconnect; + } + + @Override + protected void doStart() { + final StoppableConnectionChecker connectionChecker = new StoppableConnectionChecker(); + this.connectionChecker = connectionChecker; + connectionChecker.scheduleNextCheck(); + } + + class StoppableConnectionChecker extends NodeConnectionsService.ConnectionChecker { + @Override + protected void doRun() { + if (connectionChecker == this && shouldReconnect) { + connectDisconnectedTargets(this::scheduleNextCheck); + } else { + // Skip reconnection attempt but still schedule the next check + scheduleNextCheck(); + } + } } } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 350c6f9ae8f6b..440227436175d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1923,11 +1923,6 @@ private final class TestClusterNode { protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() { return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool); } - - @Override - protected void connectToNodesAndWait(ClusterState newClusterState) { - // don't do anything, and don't block - } } ); recoverySettings = new RecoverySettings(settings, clusterSettings); @@ -2094,7 +2089,7 @@ public void onFailure(final Exception e) { rerouteService, threadPool ); - nodeConnectionsService = new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService); + nodeConnectionsService = createTestNodeConnectionsService(clusterService.getSettings(), threadPool, transportService); final MetadataMappingService metadataMappingService = new MetadataMappingService(clusterService, indicesService); indicesClusterStateService = new IndicesClusterStateService( settings, @@ -2492,6 +2487,24 @@ protected void assertSnapshotOrGenericThread() { } } + public NodeConnectionsService createTestNodeConnectionsService( + Settings settings, + ThreadPool threadPool, + TransportService transportService + ) { + return new NodeConnectionsService(settings, threadPool, transportService) { + @Override + public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) { + // just update targetsByNode to ensure disconnect runs for these nodes + // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen + for (final DiscoveryNode discoveryNode : discoveryNodes) { + this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode)); + } + onCompletion.run(); + } + }; + } + public ClusterInfoService getMockClusterInfoService() { return clusterInfoService; } @@ -2563,10 +2576,11 @@ public void start(ClusterState initialState) { new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), null ); + coordinator.setNodeConnectionsService(nodeConnectionsService); clusterManagerService.setClusterStatePublisher(coordinator); - coordinator.start(); clusterService.getClusterApplierService().setNodeConnectionsService(nodeConnectionsService); nodeConnectionsService.start(); + coordinator.start(); clusterService.start(); indicesService.start(); indicesClusterStateService.start(); diff --git a/server/src/test/java/org/opensearch/test/NoopDiscovery.java b/server/src/test/java/org/opensearch/test/NoopDiscovery.java index 42d3f1887ab4d..c35503a556db6 100644 --- a/server/src/test/java/org/opensearch/test/NoopDiscovery.java +++ b/server/src/test/java/org/opensearch/test/NoopDiscovery.java @@ -32,6 +32,7 @@ package org.opensearch.test; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.NodeConnectionsService; import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.core.action.ActionListener; @@ -55,6 +56,11 @@ public void startInitialJoin() { } + @Override + public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) { + + } + @Override public Lifecycle.State lifecycleState() { return null; diff --git a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java index 1d734a56ef189..fdf762aa096f0 100644 --- a/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java +++ b/server/src/test/java/org/opensearch/transport/ClusterConnectionManagerTests.java @@ -320,6 +320,50 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti assertEquals(0, nodeDisconnectedCount.get()); } + public void testConnectFailsWhenDisconnectIsPending() { + AtomicInteger nodeConnectedCount = new AtomicInteger(); + AtomicInteger nodeDisconnectedCount = new AtomicInteger(); + connectionManager.addListener(new TransportConnectionListener() { + @Override + public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { + nodeConnectedCount.incrementAndGet(); + } + + @Override + public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + nodeDisconnectedCount.incrementAndGet(); + } + }); + + DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT); + ConnectionManager.ConnectionValidator validator = (c, p, l) -> l.onResponse(null); + Transport.Connection connection = new TestConnect(node); + doAnswer(invocationOnMock -> { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(connection); + return null; + }).when(transport).openConnection(eq(node), eq(connectionProfile), any(ActionListener.class)); + assertFalse(connectionManager.nodeConnected(node)); + + // Mark connection as pending disconnect, any connection attempt should fail + connectionManager.setPendingDisconnection(node); + PlainActionFuture fut = new PlainActionFuture<>(); + connectionManager.connectToNode(node, connectionProfile, validator, fut); + expectThrows(IllegalStateException.class, () -> fut.actionGet()); + + // clear the pending disconnect and assert that connection succeeds + connectionManager.clearPendingDisconnections(); + assertFalse(connectionManager.nodeConnected(node)); + PlainActionFuture.get( + future -> connectionManager.connectToNode(node, connectionProfile, validator, ActionListener.map(future, x -> null)) + ); + assertFalse(connection.isClosed()); + assertTrue(connectionManager.nodeConnected(node)); + assertEquals(1, connectionManager.size()); + assertEquals(1, nodeConnectedCount.get()); + assertEquals(0, nodeDisconnectedCount.get()); + } + private static class TestConnect extends CloseableConnection { private final DiscoveryNode node; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index b432e5411404e..3efcc538a1b25 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -55,6 +55,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterService; @@ -1150,9 +1151,12 @@ protected Optional getDisruptableMockTransport(Transpo new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ); clusterService = new ClusterService(settings, clusterSettings, clusterManagerService, clusterApplierService); - clusterService.setNodeConnectionsService( - new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService) + NodeConnectionsService nodeConnectionsService = createTestNodeConnectionsService( + clusterService.getSettings(), + threadPool, + transportService ); + clusterService.setNodeConnectionsService(nodeConnectionsService); repositoriesService = new RepositoriesService( settings, clusterService, @@ -1187,6 +1191,7 @@ protected Optional getDisruptableMockTransport(Transpo new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE), null ); + coordinator.setNodeConnectionsService(nodeConnectionsService); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( settings, @@ -1588,6 +1593,24 @@ public void onNodeAck(DiscoveryNode node, Exception e) { } } + public static NodeConnectionsService createTestNodeConnectionsService( + Settings settings, + ThreadPool threadPool, + TransportService transportService + ) { + return new NodeConnectionsService(settings, threadPool, transportService) { + @Override + public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion) { + // just update targetsByNode to ensure disconnect runs for these nodes + // we rely on disconnect to run for keeping track of pendingDisconnects and ensuring node-joins can happen + for (final DiscoveryNode discoveryNode : discoveryNodes) { + this.targetsByNode.put(discoveryNode, createConnectionTarget(discoveryNode)); + } + onCompletion.run(); + } + }; + } + static class DisruptableClusterApplierService extends ClusterApplierService { private final String nodeName; private final DeterministicTaskQueue deterministicTaskQueue; @@ -1641,11 +1664,6 @@ public void onNewClusterState(String source, Supplier clusterState } } - @Override - protected void connectToNodesAndWait(ClusterState newClusterState) { - // don't do anything, and don't block - } - @Override protected boolean applicationMayFail() { return this.applicationMayFail; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java index 1ad18bf89d5ba..4f692c7bc8f62 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/DeterministicTaskQueue.java @@ -92,6 +92,12 @@ public void runAllRunnableTasks() { } } + public void runAllRunnableTasksInEnqueuedOrder() { + while (hasRunnableTasks()) { + runTask(0); + } + } + public void runAllTasks() { while (hasDeferredTasks() || hasRunnableTasks()) { if (hasDeferredTasks() && random.nextBoolean()) { @@ -141,6 +147,11 @@ public void runRandomTask() { runTask(RandomNumbers.randomIntBetween(random, 0, runnableTasks.size() - 1)); } + public void runNextTask() { + assert hasRunnableTasks(); + runTask(0); + } + private void runTask(final int index) { final Runnable task = runnableTasks.remove(index); logger.trace("running task {} of {}: {}", index, runnableTasks.size() + 1, task); diff --git a/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java new file mode 100644 index 0000000000000..030f399a5bcc0 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/TestLogsAppender.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.layout.PatternLayout; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Test logs appender that provides functionality to extract specific logs/exception messages and wait for it to show up + * @opensearch.internal + */ +public class TestLogsAppender extends AbstractAppender { + private final List capturedLogs = new ArrayList<>(); + private final List messagesToCapture; + + public TestLogsAppender(List messagesToCapture) { + super("TestAppender", null, PatternLayout.createDefaultLayout(), false, Property.EMPTY_ARRAY); + this.messagesToCapture = messagesToCapture; + start(); + } + + @Override + public void append(LogEvent event) { + if (shouldCaptureMessage(event.getMessage().getFormattedMessage())) capturedLogs.add(event.getMessage().getFormattedMessage()); + if (event.getThrown() != null) { + if (shouldCaptureMessage(event.getThrown().toString())) capturedLogs.add(event.getThrown().toString()); + for (StackTraceElement element : event.getThrown().getStackTrace()) + if (shouldCaptureMessage(element.toString())) capturedLogs.add(element.toString()); + } + } + + public boolean shouldCaptureMessage(String log) { + return messagesToCapture.stream().anyMatch(log::contains); + } + + public List getCapturedLogs() { + return new ArrayList<>(capturedLogs); + } + + public boolean waitForLog(String expectedLog, long timeout, TimeUnit unit) { + long startTime = System.currentTimeMillis(); + long timeoutInMillis = unit.toMillis(timeout); + + while (System.currentTimeMillis() - startTime < timeoutInMillis) { + if (capturedLogs.stream().anyMatch(log -> log.contains(expectedLog))) { + return true; + } + try { + Thread.sleep(100); // Wait for 100ms before checking again + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + return false; + } + + // Clear captured logs + public void clearCapturedLogs() { + capturedLogs.clear(); + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java index 37df90fb103a3..d1e1a3e8af17c 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/StubbableConnectionManager.java @@ -123,6 +123,16 @@ public void disconnectFromNode(DiscoveryNode node) { delegate.disconnectFromNode(node); } + @Override + public void setPendingDisconnection(DiscoveryNode node) { + delegate.setPendingDisconnection(node); + } + + @Override + public void clearPendingDisconnections() { + delegate.clearPendingDisconnections(); + } + @Override public int size() { return delegate.size(); From d510b129ff66e0d77b4e7390c10bd309d14c9c1a Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Sat, 28 Sep 2024 22:36:37 -0400 Subject: [PATCH 157/357] Update Gradle to 8.10.2 (#16107) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 5c6205ebf24d4..e312a2da77d94 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15 +distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6 From 031d8ae022e47aedc8563bb7735aa582c8145bde Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 30 Sep 2024 02:38:15 +0800 Subject: [PATCH 158/357] Fix testHAProxyModeConnectionWorks is flaky (#16087) * Fix testHAProxyModeConnectionWorks is flaky Signed-off-by: Gao Binlong * Optimize health check method Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- qa/remote-clusters/docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml index cf6aefcf5c1a3..2112da17efe6e 100644 --- a/qa/remote-clusters/docker-compose.yml +++ b/qa/remote-clusters/docker-compose.yml @@ -72,3 +72,11 @@ services: - "9600" volumes: - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - opensearch-2 + healthcheck: + test: ["CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/localhost/9600"] + interval: 2s + timeout: 1s + retries: 5 + start_period: 15s From 0dba562f896a7294fc16eddf64081fe45b3aa3bb Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:34:27 +0530 Subject: [PATCH 159/357] Implementing pagination for _cat/indices API (#14718) * Implementing pagination for _cat/indices Signed-off-by: Harsh Garg --- CHANGELOG.md | 1 + .../org/opensearch/action/ActionModule.java | 14 +- .../java/org/opensearch/common/Table.java | 15 + .../java/org/opensearch/rest/RestHandler.java | 7 + .../java/org/opensearch/rest/RestRequest.java | 8 + .../rest/action/cat/RestIndicesAction.java | 163 +++++-- .../opensearch/rest/action/cat/RestTable.java | 41 +- .../rest/action/list/AbstractListAction.java | 77 ++++ .../action/list/RestIndicesListAction.java | 104 +++++ .../rest/action/list/RestListAction.java | 58 +++ .../rest/action/list/package-info.java | 12 + .../pagination/IndexPaginationStrategy.java | 185 ++++++++ .../rest/pagination/PageParams.java | 48 +++ .../opensearch/rest/pagination/PageToken.java | 42 ++ .../rest/pagination/PaginationStrategy.java | 75 ++++ .../rest/pagination/package-info.java | 12 + .../action/cat/RestIndicesActionTests.java | 70 ++- .../rest/action/cat/RestTableTests.java | 93 +++- .../IndexPaginationStrategyTests.java | 399 ++++++++++++++++++ 19 files changed, 1355 insertions(+), 69 deletions(-) create mode 100644 server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/package-info.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageParams.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageToken.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/package-info.java create mode 100644 server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 07637eaae3306..b628e9277959d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) +- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 3fe0f1dc7cb83..937d7509fe68c 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -461,6 +461,9 @@ import org.opensearch.rest.action.ingest.RestGetPipelineAction; import org.opensearch.rest.action.ingest.RestPutPipelineAction; import org.opensearch.rest.action.ingest.RestSimulatePipelineAction; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.action.list.RestListAction; import org.opensearch.rest.action.search.RestClearScrollAction; import org.opensearch.rest.action.search.RestCountAction; import org.opensearch.rest.action.search.RestCreatePitAction; @@ -802,9 +805,14 @@ private ActionFilters setupActionFilters(List actionPlugins) { public void initRestHandlers(Supplier nodesInCluster) { List catActions = new ArrayList<>(); + List listActions = new ArrayList<>(); Consumer registerHandler = handler -> { if (handler instanceof AbstractCatAction) { - catActions.add((AbstractCatAction) handler); + if (handler instanceof AbstractListAction && ((AbstractListAction) handler).isActionPaginated()) { + listActions.add((AbstractListAction) handler); + } else { + catActions.add((AbstractCatAction) handler); + } } restController.registerHandler(handler); }; @@ -980,6 +988,9 @@ public void initRestHandlers(Supplier nodesInCluster) { } registerHandler.accept(new RestTemplatesAction()); + // LIST API + registerHandler.accept(new RestIndicesListAction()); + // Point in time API registerHandler.accept(new RestCreatePitAction()); registerHandler.accept(new RestDeletePitAction()); @@ -1011,6 +1022,7 @@ public void initRestHandlers(Supplier nodesInCluster) { } } registerHandler.accept(new RestCatAction(catActions)); + registerHandler.accept(new RestListAction(listActions)); registerHandler.accept(new RestDecommissionAction()); registerHandler.accept(new RestGetDecommissionStateAction()); registerHandler.accept(new RestRemoteStoreStatsAction()); diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index da14f628efa0f..133ec3052e6c9 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -34,6 +34,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.core.common.Strings; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -59,9 +60,19 @@ public class Table { private List currentCells; private boolean inHeaders = false; private boolean withTime = false; + /** + * paginatedQueryResponse if null will imply the Table response is not paginated. + */ + private PageToken pageToken; public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; + public Table() {} + + public Table(@Nullable PageToken pageToken) { + this.pageToken = pageToken; + } + public Table startHeaders() { inHeaders = true; currentCells = new ArrayList<>(); @@ -230,6 +241,10 @@ public Map getAliasMap() { return headerAliasMap; } + public PageToken getPageToken() { + return pageToken; + } + /** * Cell in a table * diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 1139e5fc65f31..143cbd472ed07 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -125,6 +125,13 @@ default boolean allowSystemIndexAccessByDefault() { return false; } + /** + * Denotes whether the RestHandler will output paginated responses or not. + */ + default boolean isActionPaginated() { + return false; + } + static RestHandler wrapper(RestHandler delegate) { return new Wrapper(delegate); } diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 2c397f7fc6e8e..f241b567c3204 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -51,6 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; +import org.opensearch.rest.pagination.PageParams; import java.io.IOException; import java.io.InputStream; @@ -67,6 +68,9 @@ import static org.opensearch.common.unit.TimeValue.parseTimeValue; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.rest.pagination.PageParams.PARAM_NEXT_TOKEN; +import static org.opensearch.rest.pagination.PageParams.PARAM_SIZE; +import static org.opensearch.rest.pagination.PageParams.PARAM_SORT; /** * REST Request @@ -591,6 +595,10 @@ public static MediaType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } + public PageParams parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) { + return new PageParams(param(PARAM_NEXT_TOKEN), param(PARAM_SORT, defaultSortOrder), paramAsInt(PARAM_SIZE, defaultPageSize)); + } + /** * Thrown if there is an error in the content type header. * diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 9dc711f804144..1e76008ff8c64 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -50,6 +50,7 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Table; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; @@ -61,6 +62,9 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -68,9 +72,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.Spliterators; import java.util.function.Function; @@ -87,7 +93,7 @@ * * @opensearch.api */ -public class RestIndicesAction extends AbstractCatAction { +public class RestIndicesAction extends AbstractListAction { private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class); @@ -151,48 +157,70 @@ public RestResponse buildResponse(final Table table) throws Exception { new ActionListener() { @Override public void onResponse(final GetSettingsResponse getSettingsResponse) { - final GroupedActionListener groupedListener = createGroupedListener(request, 4, listener); - groupedListener.onResponse(getSettingsResponse); - // The list of indices that will be returned is determined by the indices returned from the Get Settings call. // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the get settings request might be deleted when the subsequent - // cluster - // state, cluster health and indices stats requests execute. We have to distinguish two cases: - // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the - // subsequent requests to fail. - // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests - // not to - // fail on the deleted index (as we want to ignore wildcards that cannot be resolved). - // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve - // the - // index names with the same indices options that we used for the initial cluster state request (strictExpand). - sendIndicesStatsRequest( - indices, - subRequestIndicesOptions, - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); + // Indices that were successfully resolved during the get settings request might be deleted when the + // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases: + // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we + // want the subsequent requests to fail. + // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent + // requests not to fail on the deleted index (as we want to ignore wildcards that cannot be resolved). + // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests + // re-resolve the index names with the same indices options that we used for the initial cluster state + // request (strictExpand). sendClusterStateRequest( indices, subRequestIndicesOptions, local, clusterManagerNodeTimeout, client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendClusterHealthRequest( - indices, - subRequestIndicesOptions, - local, - clusterManagerNodeTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + new ActionListener() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); + // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from + // rest request and unresolved, while for paginated queries, it would be a list of indices + // already resolved by ClusterStateRequest and to be displayed in a page. + final String[] indicesToBeQueried = Objects.isNull(paginationStrategy) + ? indices + : paginationStrategy.getRequestedEntities().toArray(new String[0]); + final GroupedActionListener groupedListener = createGroupedListener( + request, + 4, + listener, + indicesToBeQueried, + Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken() + ); + groupedListener.onResponse(getSettingsResponse); + groupedListener.onResponse(clusterStateResponse); + + sendIndicesStatsRequest( + indicesToBeQueried, + subRequestIndicesOptions, + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + + sendClusterHealthRequest( + indicesToBeQueried, + subRequestIndicesOptions, + local, + clusterManagerNodeTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } ); } @@ -203,6 +231,7 @@ public void onFailure(final Exception e) { } ); }; + } /** @@ -287,7 +316,9 @@ private void sendIndicesStatsRequest( private GroupedActionListener createGroupedListener( final RestRequest request, final int size, - final ActionListener listener + final ActionListener
listener, + final String[] indicesToBeQueried, + final PageToken pageToken ) { return new GroupedActionListener<>(new ActionListener>() { @Override @@ -311,7 +342,15 @@ public void onResponse(final Collection responses) { IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); Map indicesStats = statsResponse.getIndices(); - Table responseTable = buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates); + Table responseTable = buildTable( + request, + indicesSettings, + indicesHealths, + indicesStats, + indicesStates, + getTableIterator(indicesToBeQueried, indicesSettings), + pageToken + ); listener.onResponse(responseTable); } catch (Exception e) { onFailure(e); @@ -340,7 +379,11 @@ protected Set responseParams() { @Override protected Table getTableWithHeader(final RestRequest request) { - Table table = new Table(); + return getTableWithHeader(request, null); + } + + protected Table getTableWithHeader(final RestRequest request, final PageToken pageToken) { + Table table = new Table(pageToken); table.startHeaders(); table.addCell("health", "alias:h;desc:current health status"); table.addCell("status", "alias:s;desc:open/close status"); @@ -704,22 +747,27 @@ protected Table getTableWithHeader(final RestRequest request) { } // package private for testing - Table buildTable( + protected Table buildTable( final RestRequest request, final Map indicesSettings, final Map indicesHealths, final Map indicesStats, - final Map indicesMetadatas + final Map indicesMetadatas, + final Iterator> tableIterator, + final PageToken pageToken ) { - final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request); + final Table table = getTableWithHeader(request, pageToken); + + while (tableIterator.hasNext()) { + final Tuple tuple = tableIterator.next(); + String indexName = tuple.v1(); + Settings settings = tuple.v2(); - indicesSettings.forEach((indexName, settings) -> { if (indicesMetadatas.containsKey(indexName) == false) { // the index exists in the Get Indices response but is not present in the cluster state: // it is likely that the index was deleted in the meanwhile, so we ignore it. - return; + continue; } final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); @@ -748,7 +796,7 @@ Table buildTable( skip = ClusterHealthStatus.RED != healthStatusFilter; } if (skip) { - return; + continue; } } @@ -982,7 +1030,8 @@ Table buildTable( table.addCell(searchThrottled); table.endRow(); - }); + + } return table; } @@ -991,4 +1040,34 @@ Table buildTable( private static A extractResponse(final Collection responses, Class c) { return (A) responses.stream().filter(c::isInstance).findFirst().get(); } + + @Override + public boolean isActionPaginated() { + return false; + } + + protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return null; + } + + /** + * Provides the iterator to be used for building the response table. + */ + protected Iterator> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + final Iterator settingsMapIter = indexSettingsMap.keySet().iterator(); + + @Override + public boolean hasNext() { + return settingsMapIter.hasNext(); + } + + @Override + public Tuple next() { + String index = settingsMapIter.next(); + return new Tuple<>(index, indexSettingsMap.get(index)); + } + }; + } + } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 4f1090b163ee6..d622dd7a956f4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -58,8 +58,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; +import static org.opensearch.rest.pagination.PageToken.PAGINATED_RESPONSE_NEXT_TOKEN_KEY; + /** * a REST table * @@ -87,8 +90,37 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel RestRequest request = channel.request(); XContentBuilder builder = channel.newBuilder(); List displayHeaders = buildDisplayHeaders(table, request); + if (Objects.nonNull(table.getPageToken())) { + buildPaginatedXContentBuilder(table, request, builder, displayHeaders); + } else { + builder.startArray(); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); + } + return new BytesRestResponse(RestStatus.OK, builder); + } + + private static void buildPaginatedXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { + assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) : "Paginated element is required in-case of paginated responses"; + builder.startObject(); + builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken()); + builder.startArray(table.getPageToken().getPaginatedEntity()); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); + builder.endObject(); + } - builder.startArray(); + private static void addRowsToXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { List rowOrder = getRowOrder(table, request); for (Integer row : rowOrder) { builder.startObject(); @@ -97,8 +129,6 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel } builder.endObject(); } - builder.endArray(); - return new BytesRestResponse(RestStatus.OK, builder); } public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException { @@ -136,6 +166,11 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann } out.append("\n"); } + // Adding a new row for next_token, in the response if the table is paginated. + if (Objects.nonNull(table.getPageToken())) { + out.append("next_token" + " " + table.getPageToken().getNextToken()); + out.append("\n"); + } out.close(); return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes()); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java new file mode 100644 index 0000000000000..f3d6d6653a550 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.AbstractCatAction; +import org.opensearch.rest.pagination.PageParams; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; + +/** + * Base Transport action class for _list API. + * Serves as a base class for APIs wanting to support pagination. + * Existing _cat APIs can refer {@link org.opensearch.rest.action.cat.RestIndicesAction}. + * @opensearch.api + */ +public abstract class AbstractListAction extends AbstractCatAction { + + private static final int DEFAULT_PAGE_SIZE = 100; + protected PageParams pageParams; + + protected abstract void documentation(StringBuilder sb); + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + boolean helpWanted = request.paramAsBoolean("help", false); + if (helpWanted || isActionPaginated() == false) { + return super.prepareRequest(request, client); + } + this.pageParams = validateAndGetPageParams(request); + assert Objects.nonNull(pageParams) : "pageParams can not be null for paginated queries"; + return doCatRequest(request, client); + } + + @Override + public boolean isActionPaginated() { + return true; + } + + /** + * + * @return Metadata that can be extracted out from the rest request. Query params supported by the action specific + * to pagination along with any respective validations to be added here. + */ + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = restRequest.parsePaginatedQueryParams(defaultSort(), defaultPageSize()); + // validating pageSize + if (pageParams.getSize() <= 0) { + throw new IllegalArgumentException("size must be greater than zero"); + } + // Validating sort order + if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) { + throw new IllegalArgumentException("value of sort can either be asc or desc"); + } + return pageParams; + } + + protected int defaultPageSize() { + return DEFAULT_PAGE_SIZE; + } + + protected String defaultSort() { + return PARAM_ASC_SORT_VALUE; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java new file mode 100644 index 0000000000000..ad5c58c86ce90 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.settings.Settings; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.RestIndicesAction; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageParams; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * _list API action to output indices in pages. + * + * @opensearch.api + */ +public class RestIndicesListAction extends RestIndicesAction { + + private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; + private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; + + @Override + public List routes() { + return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}"))); + } + + @Override + public String getName() { + return "list_indices_action"; + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_list/indices\n"); + sb.append("/_list/indices/{index}\n"); + } + + @Override + public boolean isActionPaginated() { + return true; + } + + @Override + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = super.validateAndGetPageParams(restRequest); + // validate max supported pageSize + if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE) { + throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE + "]"); + } + // Next Token in the request will be validated by the IndexStrategyToken itself. + if (Objects.nonNull(pageParams.getRequestedToken())) { + IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken()); + } + return pageParams; + } + + protected int defaultPageSize() { + return DEFAULT_LIST_INDICES_PAGE_SIZE; + } + + @Override + protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return new IndexPaginationStrategy(pageParams, clusterStateResponse.getState()); + } + + // Public for testing + @Override + public Iterator> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + int indexPos = 0; + + @Override + public boolean hasNext() { + while (indexPos < indices.length && indexSettingsMap.containsKey(indices[indexPos]) == false) { + indexPos++; + } + return indexPos < indices.length; + } + + @Override + public Tuple next() { + Tuple tuple = new Tuple<>(indices[indexPos], indexSettingsMap.get(indices[indexPos])); + indexPos++; + return tuple; + } + }; + } +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java new file mode 100644 index 0000000000000..4b8551ea7e14a --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * Base _list API endpoint + * + * @opensearch.api + */ +public class RestListAction extends BaseRestHandler { + + private static final String LIST = ":‑|"; + private static final String LIST_NL = LIST + "\n"; + private final String HELP; + + public RestListAction(List listActions) { + StringBuilder sb = new StringBuilder(); + sb.append(LIST_NL); + for (AbstractListAction listAction : listActions) { + listAction.documentation(sb); + } + HELP = sb.toString(); + } + + @Override + public List routes() { + return singletonList(new Route(GET, "/_list")); + } + + @Override + public String getName() { + return "list_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP)); + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/package-info.java b/server/src/main/java/org/opensearch/rest/action/list/package-info.java new file mode 100644 index 0000000000000..8d6563ff9b344 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * {@link org.opensearch.rest.RestHandler}s for actions that list out results in chunks of pages. + */ +package org.opensearch.rest.action.list; diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java new file mode 100644 index 0000000000000..f89ab14e4b24d --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Nullable; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; + +/** + * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. + * The strategy considers create timestamps of indices as the keys to iterate over pages. + * + * @opensearch.internal + */ +public class IndexPaginationStrategy implements PaginationStrategy { + private static final String DEFAULT_INDICES_PAGINATED_ENTITY = "indices"; + + private static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); + } + return Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); + }; + private static final Comparator DESC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); + } + return Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()); + }; + + private final PageToken pageToken; + private final List requestedIndices; + + public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) { + // Get list of indices metadata sorted by their creation time and filtered by the last sent index + List sortedIndices = PaginationStrategy.getSortedIndexMetadata( + clusterState, + getMetadataFilter(pageParams.getRequestedToken(), pageParams.getSort()), + PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + // Trim sortedIndicesList to get the list of indices metadata to be sent as response + List metadataSublist = getMetadataSubList(sortedIndices, pageParams.getSize()); + // Get list of index names from the trimmed metadataSublist + this.requestedIndices = metadataSublist.stream().map(metadata -> metadata.getIndex().getName()).collect(Collectors.toList()); + this.pageToken = getResponseToken( + pageParams.getSize(), + sortedIndices.size(), + metadataSublist.isEmpty() ? null : metadataSublist.get(metadataSublist.size() - 1) + ); + } + + private static Predicate getMetadataFilter(String requestedTokenStr, String sortOrder) { + boolean isAscendingSort = sortOrder.equals(PARAM_ASC_SORT_VALUE); + IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() + ? null + : new IndexStrategyToken(requestedTokenStr); + if (Objects.isNull(requestedToken)) { + return indexMetadata -> true; + } + return metadata -> { + if (metadata.getIndex().getName().equals(requestedToken.lastIndexName)) { + return false; + } else if (metadata.getCreationDate() == requestedToken.lastIndexCreationTime) { + return isAscendingSort + ? metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) > 0 + : metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) < 0; + } + return isAscendingSort + ? metadata.getCreationDate() > requestedToken.lastIndexCreationTime + : metadata.getCreationDate() < requestedToken.lastIndexCreationTime; + }; + } + + private List getMetadataSubList(List sortedIndices, final int pageSize) { + if (sortedIndices.isEmpty()) { + return new ArrayList<>(); + } + return sortedIndices.subList(0, Math.min(pageSize, sortedIndices.size())); + } + + private PageToken getResponseToken(final int pageSize, final int totalIndices, IndexMetadata lastIndex) { + if (totalIndices <= pageSize) { + return new PageToken(null, DEFAULT_INDICES_PAGINATED_ENTITY); + } + return new PageToken( + new IndexStrategyToken(lastIndex.getCreationDate(), lastIndex.getIndex().getName()).generateEncryptedToken(), + DEFAULT_INDICES_PAGINATED_ENTITY + ); + } + + @Override + @Nullable + public PageToken getResponseToken() { + return pageToken; + } + + @Override + public List getRequestedEntities() { + return Objects.isNull(requestedIndices) ? new ArrayList<>() : requestedIndices; + } + + /** + * TokenParser to be used by {@link IndexPaginationStrategy}. + * Token would look like: CreationTimeOfLastRespondedIndex + | + NameOfLastRespondedIndex + */ + public static class IndexStrategyToken { + + private static final String JOIN_DELIMITER = "|"; + private static final String SPLIT_REGEX = "\\|"; + private static final int CREATE_TIME_POS_IN_TOKEN = 0; + private static final int INDEX_NAME_POS_IN_TOKEN = 1; + + /** + * Represents creation times of last index which was displayed in the page. + * Used to identify the new start point in case the indices get created/deleted while queries are executed. + */ + private final long lastIndexCreationTime; + + /** + * Represents name of the last index which was displayed in the page. + * Used to identify whether the sorted list of indices has changed or not. + */ + private final String lastIndexName; + + public IndexStrategyToken(String requestedTokenString) { + // TODO: Avoid validating the requested token multiple times while calling from Rest and/or Transport layer. + validateIndexStrategyToken(requestedTokenString); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + this.lastIndexCreationTime = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + this.lastIndexName = decryptedTokenElements[INDEX_NAME_POS_IN_TOKEN]; + } + + public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { + Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); + this.lastIndexCreationTime = creationTimeOfLastRespondedIndex; + this.lastIndexName = nameOfLastRespondedIndex; + } + + public String generateEncryptedToken() { + return PaginationStrategy.encryptStringToken(String.join(JOIN_DELIMITER, String.valueOf(lastIndexCreationTime), lastIndexName)); + } + + /** + * Will perform simple validations on token received in the request. + * Token should be base64 encoded, and should contain the expected number of elements separated by "|". + * Timestamps should also be a valid long. + * + * @param requestedTokenStr string denoting the encoded token requested by the user. + */ + public static void validateIndexStrategyToken(String requestedTokenStr) { + Objects.requireNonNull(requestedTokenStr, "requestedTokenString can not be null"); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenStr); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + if (decryptedTokenElements.length != 2) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + try { + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + if (creationTimeOfLastRespondedIndex <= 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageParams.java b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java new file mode 100644 index 0000000000000..9b2074bc3fed0 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.common.annotation.PublicApi; + +/** + * + * Class specific to paginated queries, which will contain common query params required by a paginated API. + */ +@PublicApi(since = "3.0.0") +public class PageParams { + + public static final String PARAM_SORT = "sort"; + public static final String PARAM_NEXT_TOKEN = "next_token"; + public static final String PARAM_SIZE = "size"; + public static final String PARAM_ASC_SORT_VALUE = "asc"; + public static final String PARAM_DESC_SORT_VALUE = "desc"; + + private final String requestedTokenStr; + private final String sort; + private final int size; + + public PageParams(String requestedToken, String sort, int size) { + this.requestedTokenStr = requestedToken; + this.sort = sort; + this.size = size; + } + + public String getSort() { + return sort; + } + + public String getRequestedToken() { + return requestedTokenStr; + } + + public int getSize() { + return size; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java new file mode 100644 index 0000000000000..d62e1be695715 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +/** + * Pagination response metadata for a paginated query. + * @opensearch.internal + */ +public class PageToken { + + public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; + + /** + * String denoting the next_token of paginated response, which will be used to fetch next page (if any). + */ + private final String nextToken; + + /** + * String denoting the element which is being paginated (for e.g. shards, indices..). + */ + private final String paginatedEntity; + + public PageToken(String nextToken, String paginatedElement) { + assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; + this.nextToken = nextToken; + this.paginatedEntity = paginatedElement; + } + + public String getNextToken() { + return nextToken; + } + + public String getPaginatedEntity() { + return paginatedEntity; + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java new file mode 100644 index 0000000000000..7f9825a7cc09b --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; + +import java.util.Base64; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Interface to be implemented by any strategy getting used for paginating rest responses. + * + * @opensearch.internal + */ +public interface PaginationStrategy { + + String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = + "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; + + /** + * + * @return Base64 encoded string, which can be used to fetch next page of response. + */ + PageToken getResponseToken(); + + /** + * + * @return List of elements fetched corresponding to the store and token received by the strategy. + */ + List getRequestedEntities(); + + /** + * + * Utility method to get list of indices filtered as per {@param filterPredicate} and the sorted according to {@param comparator}. + */ + static List getSortedIndexMetadata( + final ClusterState clusterState, + Predicate filterPredicate, + Comparator comparator + ) { + return clusterState.metadata().indices().values().stream().filter(filterPredicate).sorted(comparator).collect(Collectors.toList()); + } + + static String encryptStringToken(String tokenString) { + if (Objects.isNull(tokenString)) { + return null; + } + return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8)); + } + + static String decryptStringToken(String encTokenString) { + if (Objects.isNull(encTokenString)) { + return null; + } + try { + return new String(Base64.getDecoder().decode(encTokenString), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/package-info.java b/server/src/main/java/org/opensearch/rest/pagination/package-info.java new file mode 100644 index 0000000000000..324b8a6c46f88 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Exposes utilities for Rest actions to paginate responses. + */ +package org.opensearch.rest.pagination; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 96b1c75371697..1d1b509ae94e5 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -47,9 +47,13 @@ import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Before; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -63,13 +67,14 @@ public class RestIndicesActionTests extends OpenSearchTestCase { - public void testBuildTable() { - final int numIndices = randomIntBetween(3, 20); - final Map indicesSettings = new LinkedHashMap<>(); - final Map indicesMetadatas = new LinkedHashMap<>(); - final Map indicesHealths = new LinkedHashMap<>(); - final Map indicesStats = new LinkedHashMap<>(); + final Map indicesSettings = new LinkedHashMap<>(); + final Map indicesMetadatas = new LinkedHashMap<>(); + final Map indicesHealths = new LinkedHashMap<>(); + final Map indicesStats = new LinkedHashMap<>(); + @Before + public void setup() { + final int numIndices = randomIntBetween(3, 20); for (int i = 0; i < numIndices; i++) { String indexName = "index-" + i; @@ -136,11 +141,59 @@ public void testBuildTable() { } } } + } + public void testBuildTable() { final RestIndicesAction action = new RestIndicesAction(); - final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas); + final Table table = action.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + action.getTableIterator(new String[0], indicesSettings), + null + ); // now, verify the table is correct + assertNotNull(table); + + assertTableHeaders(table); + + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size())); + assertTableRows(table); + } + + public void testBuildPaginatedTable() { + final RestIndicesAction action = new RestIndicesAction(); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(); + List indicesList = new ArrayList<>(indicesMetadatas.keySet()); + // Using half of the indices from metadata list for a page + String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); + PageToken pageToken = new PageToken("foo", "indices"); + final Table table = action.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + indicesListAction.getTableIterator(indicesToBeQueried, indicesSettings), + pageToken + ); + + // verifying table + assertNotNull(table); + assertTableHeaders(table); + assertNotNull(table.getPageToken()); + assertEquals(pageToken.getNextToken(), table.getPageToken().getNextToken()); + assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity()); + + // Table should only contain the indices present in indicesToBeQueried + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() / 2)); + assertTableRows(table); + } + + private void assertTableHeaders(Table table) { List headers = table.getHeaders(); assertThat(headers.get(0).value, equalTo("health")); assertThat(headers.get(1).value, equalTo("status")); @@ -148,9 +201,10 @@ public void testBuildTable() { assertThat(headers.get(3).value, equalTo("uuid")); assertThat(headers.get(4).value, equalTo("pri")); assertThat(headers.get(5).value, equalTo("rep")); + } + private void assertTableRows(Table table) { final List> rows = table.getRows(); - assertThat(rows.size(), equalTo(indicesMetadatas.size())); for (final List row : rows) { final String indexName = (String) row.get(2).value; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java index 8183cb1d3b910..a82e563d70273 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java @@ -37,6 +37,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestResponse; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; @@ -64,9 +65,14 @@ public class RestTableTests extends OpenSearchTestCase { private static final String ACCEPT = "Accept"; private static final String TEXT_PLAIN = "text/plain; charset=UTF-8"; private static final String TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\n"; + private static final String PAGINATED_TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\nnext_token foo\n"; private static final String JSON_TABLE_BODY = "[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]"; + private static final String PAGINATED_JSON_TABLE_BODY = + "{\"next_token\":\"foo\",\"entities\":[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]}"; private static final String YAML_TABLE_BODY = "---\n" + "- bulk.foo: \"foo\"\n" + " bulk.bar: \"foo\"\n" @@ -76,6 +82,17 @@ public class RestTableTests extends OpenSearchTestCase { + " invalidAliasesBulk: \"foo\"\n" + " timestamp: \"foo\"\n" + " epoch: \"foo\"\n"; + private static final String PAGINATED_YAML_TABLE_BODY = "---\n" + + "next_token: \"foo\"\n" + + "entities:\n" + + "- bulk.foo: \"foo\"\n" + + " bulk.bar: \"foo\"\n" + + " aliasedBulk: \"foo\"\n" + + " aliasedSecondBulk: \"foo\"\n" + + " unmatched: \"foo\"\n" + + " invalidAliasesBulk: \"foo\"\n" + + " timestamp: \"foo\"\n" + + " epoch: \"foo\"\n"; private Table table; private FakeRestRequest restRequest; @@ -83,20 +100,7 @@ public class RestTableTests extends OpenSearchTestCase { public void setup() { restRequest = new FakeRestRequest(); table = new Table(); - table.startHeaders(); - table.addCell("bulk.foo", "alias:f;desc:foo"); - table.addCell("bulk.bar", "alias:b;desc:bar"); - // should be matched as well due to the aliases - table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); - table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); - // no match - table.addCell("unmatched", "alias:un.matched;desc:bar"); - // invalid alias - table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); - // timestamp - table.addCell("timestamp", "alias:ts"); - table.addCell("epoch", "alias:t"); - table.endHeaders(); + addHeaders(table); } public void testThatDisplayHeadersSupportWildcards() throws Exception { @@ -121,10 +125,28 @@ public void testThatWeUseTheAcceptHeaderJson() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderJsonForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), + APPLICATION_JSON, + PAGINATED_JSON_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderYaml() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderYamlForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), + APPLICATION_YAML, + PAGINATED_YAML_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderSmile() throws Exception { assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE); } @@ -137,6 +159,15 @@ public void testThatWeUseTheAcceptHeaderText() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderTextForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), + TEXT_PLAIN, + PAGINATED_TEXT_TABLE_BODY, + getPaginatedTable() + ); + } + public void testIgnoreContentType() throws Exception { assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY); } @@ -261,6 +292,10 @@ public void testMultiSort() { } private RestResponse assertResponseContentType(Map> headers, String mediaType) throws Exception { + return assertResponseContentType(headers, mediaType, table); + } + + private RestResponse assertResponseContentType(Map> headers, String mediaType, Table table) throws Exception { FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build(); table.startRow(); table.addCell("foo"); @@ -282,7 +317,11 @@ public void sendResponse(RestResponse response) {} } private void assertResponse(Map> headers, String mediaType, String body) throws Exception { - RestResponse response = assertResponseContentType(headers, mediaType); + assertResponse(headers, mediaType, body, table); + } + + private void assertResponse(Map> headers, String mediaType, String body, Table table) throws Exception { + RestResponse response = assertResponseContentType(headers, mediaType, table); assertThat(response.content().utf8ToString(), equalTo(body)); } @@ -294,4 +333,28 @@ private List getHeaderNames(List headers) { return headerNames; } + + private Table getPaginatedTable() { + PageToken pageToken = new PageToken("foo", "entities"); + Table paginatedTable = new Table(pageToken); + addHeaders(paginatedTable); + return paginatedTable; + } + + private void addHeaders(Table table) { + table.startHeaders(); + table.addCell("bulk.foo", "alias:f;desc:foo"); + table.addCell("bulk.bar", "alias:b;desc:bar"); + // should be matched as well due to the aliases + table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); + table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); + // no match + table.addCell("unmatched", "alias:un.matched;desc:bar"); + // invalid alias + table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); + // timestamp + table.addCell("timestamp", "alias:ts"); + table.addCell("epoch", "alias:t"); + table.endHeaders(); + } } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java new file mode 100644 index 0000000000000..01464b489e26e --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java @@ -0,0 +1,399 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.test.OpenSearchTestCase; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; + +public class IndexPaginationStrategyTests extends OpenSearchTestCase { + + public void testRetrieveAllIndicesWithVaryingPageSize() { + List indexNumberList = new ArrayList<>(); + final int totalIndices = 100; + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + // creating a cluster state with 100 indices + Collections.shuffle(indexNumberList, getRandom()); + ClusterState clusterState = getRandomClusterState(indexNumberList); + + // Checking pagination response for different pageSizes, which has a mix of even and odd numbers + // to ensure number of indices in last page is not always equal to pageSize. + List pageSizeList = List.of(1, 6, 10, 13); + List sortOrderList = List.of(PARAM_ASC_SORT_VALUE, PARAM_DESC_SORT_VALUE); + for (String sortOrder : sortOrderList) { + for (int pageSize : pageSizeList) { + String requestedToken = null; + int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); + int indicesRemaining = totalIndices; + for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { + PageParams pageParams = new PageParams(requestedToken, sortOrder, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + if (pageNumber < totalPagesToFetch) { + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + } else { + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // Asserting all the indices received + int responseItr = 0; + if (PARAM_ASC_SORT_VALUE.equals(sortOrder)) { + for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min( + 100, + pageNumber * pageSize + ); indexNumber++) { + assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } else { + int endIndexNumberForPage = Math.max(indicesRemaining - pageSize, 0); + for (; indicesRemaining > endIndexNumberForPage; indicesRemaining--) { + assertEquals("test-index-" + indicesRemaining, paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } + assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); + } + } + } + } + + public void testRetrieveAllIndicesInAscOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 10; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(20, indexNumberList.size()); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), 5); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices + numIndicesToCreate - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices + numIndicesToCreate - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // all the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertTrue(indicesFetched.contains("test-index-" + indexNumber)); + } + } + + public void testRetrieveAllIndicesInDescOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 9; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_DESC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(0, 80); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), (totalIndices - numIndicesToDelete) % pageSize); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // none of the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertFalse(indicesFetched.contains("test-index-" + indexNumber)); + } + } + + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedAscOrder() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4) + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + + // Adding index5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-2 which has already been displayed, still test-index-3 should get displayed + clusterState = deleteIndexFromClusterState(clusterState, 2); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query + // instead test-index-5 should now get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 4); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, false); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + + } + + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedDescOrder() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PageParams pageParams = new PageParams(null, PARAM_DESC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + + // adding test-index-5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-3 which has already been displayed, still index2 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 3); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 1, true); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + + // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. + clusterState = deleteIndexFromClusterState(clusterState, 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertPaginationResult(paginationStrategy, 0, false); + } + + public void testRetrieveIndicesWithMultipleDeletionsAtOnceAscOrder() { + // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // executing next query without any changes to clusterState + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // Deleting test-index-1, test-index-2 & test-index-3 and executing next query. test-index-4 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 1); + clusterState = deleteIndexFromClusterState(clusterState, 2); + clusterState = deleteIndexFromClusterState(clusterState, 3); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + + // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + + public void testRetrieveIndicesWithTokenModifiedToQueryBeyondTotal() { + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 10); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(10, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + // creating a token with last sent index as test-index-10 + String token = clusterState.metadata().indices().get("test-index-10").getCreationDate() + "|" + "test-index-10"; + pageParams = new PageParams(Base64.getEncoder().encodeToString(token.getBytes(UTF_8)), PARAM_ASC_SORT_VALUE, 10); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(0, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + + public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { + try { + new IndexPaginationStrategy.IndexStrategyToken(null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("requestedTokenString can not be null"); + } + } + + public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { + assertThrows(OpenSearchParseException.class, () -> new IndexPaginationStrategy.IndexStrategyToken("3%4%5")); + } + + public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543")) + ); + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) + ); + } + + public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index")) + ); + } + + public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { + try { + new IndexPaginationStrategy.IndexStrategyToken(1234l, null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("index name should be provided"); + } + } + + /** + * @param indexNumbers would be used to create indices having names with integer appended after foo, like foo1, foo2. + * @return random clusterState consisting of indices having their creation times set to the integer used to name them. + */ + private ClusterState getRandomClusterState(List indexNumbers) { + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().build()) + .routingTable(RoutingTable.builder().build()) + .build(); + for (Integer indexNumber : indexNumbers) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + return clusterState; + } + + private ClusterState addIndexToClusterState(ClusterState clusterState, int indexNumber) { + IndexMetadata indexMetadata = IndexMetadata.builder("test-index-" + indexNumber) + .settings( + settings(Version.CURRENT).put(SETTING_CREATION_DATE, Instant.now().plus(indexNumber, ChronoUnit.SECONDS).toEpochMilli()) + ) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + IndexRoutingTable.Builder indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()); + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder(clusterState.routingTable()).add(indexRoutingTableBuilder).build()) + .build(); + } + + private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int indexNumber) { + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).remove("test-index-" + indexNumber)) + .routingTable(RoutingTable.builder(clusterState.routingTable()).remove("test-index-" + indexNumber).build()) + .build(); + } + + private void assertPaginationResult(IndexPaginationStrategy paginationStrategy, int expectedEntities, boolean tokenExpected) { + assertNotNull(paginationStrategy); + assertEquals(expectedEntities, paginationStrategy.getRequestedEntities().size()); + assertNotNull(paginationStrategy.getResponseToken()); + assertEquals(tokenExpected, Objects.nonNull(paginationStrategy.getResponseToken().getNextToken())); + } + +} From 1bddf2f7255cf5facf17568a631cb605d78058a1 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 30 Sep 2024 10:29:41 +0530 Subject: [PATCH 160/357] Bugfix in RemoteFsTimestampAwareTranslog.trimUnreferencedReaders (#16078) Signed-off-by: Sachin Kale --- ...rePinnedTimestampsGarbageCollectionIT.java | 75 +++++++++++++ .../snapshots/DeleteSnapshotV2IT.java | 102 ++++++++++++++++++ .../RemoteFsTimestampAwareTranslog.java | 2 +- .../index/translog/RemoteFsTranslog.java | 8 ++ 4 files changed, 186 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java index 0a2668c60d3bd..08ece7df457cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java @@ -8,6 +8,7 @@ package org.opensearch.remotestore; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; @@ -32,6 +33,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsGarbageCollectionIT extends RemoteStoreBaseIntegTestCase { @@ -288,6 +290,79 @@ public void testLiveIndexWithPinnedTimestamps() throws Exception { }); } + public void testLiveIndexWithPinnedTimestampsMultiplePrimaryTerms() throws Exception { + prepareCluster(1, 2, Settings.EMPTY); + Settings indexSettings = Settings.builder() + .put(remoteStoreIndexSettings(1, 1)) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 3) + .build(); + createIndex(INDEX_NAME, indexSettings); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + int numDocs = randomIntBetween(5, 10); + for (int i = 0; i < numDocs; i++) { + keepPinnedTimestampSchedulerUpdated(); + indexSingleDoc(INDEX_NAME, true); + if (i == 2) { + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueMinutes(1)); + remoteStorePinnedTimestampService.pinTimestamp(System.currentTimeMillis(), "xyz", noOpActionListener); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + } + } + + ingestDocs(); + + internalCluster().restartNode(primaryNodeName(INDEX_NAME)); + ensureGreen(INDEX_NAME); + + ingestDocs(); + + String translogPathFixedPrefix = RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.get(getNodeSettings()); + String shardDataPath = getShardLevelBlobPath( + client(), + INDEX_NAME, + BlobPath.cleanPath(), + "0", + TRANSLOG, + DATA, + translogPathFixedPrefix + ).buildAsString(); + Path translogDataPath = Path.of(translogRepoPath + "/" + shardDataPath + "/1"); + + assertBusy(() -> { + List dataFiles = Files.list(translogDataPath).collect(Collectors.toList()); + assertFalse(dataFiles.isEmpty()); + }); + } + + private void ingestDocs() { + int numDocs = randomIntBetween(15, 20); + for (int i = 0; i < numDocs; i++) { + indexSingleDoc(INDEX_NAME, false); + } + + assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()); + flushAndRefresh(INDEX_NAME); + + int numDocsPostFailover = randomIntBetween(15, 20); + for (int i = 0; i < numDocsPostFailover; i++) { + indexSingleDoc(INDEX_NAME, false); + } + + flushAndRefresh(INDEX_NAME); + assertNoFailures(client().admin().indices().prepareRefresh(INDEX_NAME).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get()); + } + public void testIndexDeletionNoPinnedTimestamps() throws Exception { prepareCluster(1, 1, Settings.EMPTY); Settings indexSettings = Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index c4e3a478c8540..7b2ad2bccd2b1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -312,6 +313,107 @@ public void testRemoteStoreCleanupForDeletedIndexForSnapshotV2MultipleSnapshots( // translogPostDeletionOfSnapshot1.size()), 60, TimeUnit.SECONDS); } + public void testRemoteStoreCleanupMultiplePrimaryOnSnapshotDeletion() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + Settings settings = remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath); + settings = Settings.builder() + .put(settings) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED.toString()) + .build(); + String clusterManagerName = internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(3, settings); + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + ensureStableCluster(4); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + clusterManagerName + ); + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowV2(snapshotRepoPath)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = Settings.builder() + .put(getRemoteStoreBackedIndexSettings()) + .put(INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.getKey(), 2) + .build(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + ensureGreen(remoteStoreEnabledIndexName); + + // Create 2 snapshots for primary term 1 + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap1"); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap2"); + + // Restart current primary to change the primary term + internalCluster().restartNode(primaryNodeName(remoteStoreEnabledIndexName)); + ensureGreen(remoteStoreEnabledIndexName); + + // Create 2 snapshots for primary term 2 + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap3"); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + createSnapshot(snapshotRepoName, "snap4"); + + String indexUUID = client().admin() + .indices() + .prepareGetSettings(remoteStoreEnabledIndexName) + .get() + .getSetting(remoteStoreEnabledIndexName, IndexMetadata.SETTING_INDEX_UUID); + + Path indexPath = Path.of(String.valueOf(remoteStoreRepoPath), indexUUID); + Path shardPath = Path.of(String.valueOf(indexPath), "0"); + Path translogPath = Path.of(String.valueOf(shardPath), "translog", "data", "1"); + + // Deleting snap1 will still keep files in primary term 1 due to snap2 + deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap1"); + assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0); + + // Deleting snap2 will not remove primary term 1 as we need to trigger trimUnreferencedReaders once + deleteSnapshot(clusterManagerClient, snapshotRepoName, "snap2"); + assertTrue(RemoteStoreBaseIntegTestCase.getFileCount(translogPath) > 0); + + // Index a doc to trigger trimUnreferencedReaders + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + keepPinnedTimestampSchedulerUpdated(); + indexRandomDocs(remoteStoreEnabledIndexName, 5); + + assertBusy(() -> assertFalse(Files.exists(translogPath)), 30, TimeUnit.SECONDS); + } + + private void createSnapshot(String repoName, String snapshotName) { + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .get(); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName)); + } + + private void deleteSnapshot(Client clusterManagerClient, String repoName, String snapshotName) { + AcknowledgedResponse deleteSnapshotResponse = clusterManagerClient.admin() + .cluster() + .prepareDeleteSnapshot(repoName, snapshotName) + .get(); + assertAcked(deleteSnapshotResponse); + } + private Settings snapshotV2Settings(Path remoteStoreRepoPath) { Settings settings = Settings.builder() .put(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index e61a9606175ee..ede422ea3c4f7 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -121,7 +121,7 @@ public void trimUnreferencedReaders() throws IOException { protected void trimUnreferencedReaders(boolean indexDeleted, boolean trimLocal) throws IOException { if (trimLocal) { // clean up local translog files and updates readers - super.trimUnreferencedReaders(); + super.trimUnreferencedReaders(true); } // Update file tracker to reflect local translog state diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index 80760ed6ddf12..f5a9ed8ed9362 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -556,9 +556,17 @@ protected Releasable drainSync() { @Override public void trimUnreferencedReaders() throws IOException { + trimUnreferencedReaders(false); + } + + protected void trimUnreferencedReaders(boolean onlyTrimLocal) throws IOException { // clean up local translog files and updates readers super.trimUnreferencedReaders(); + if (onlyTrimLocal) { + return; + } + // This is to ensure that after the permits are acquired during primary relocation, there are no further modification on remote // store. if (startedPrimarySupplier.getAsBoolean() == false || pauseSync.get()) { From 7ba8b78bfd4a3d97876f5fe34c354eed3ad98954 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:47:55 +0530 Subject: [PATCH 161/357] Correct the version post backport to 2.18 instead of current (#16127) Signed-off-by: Gaurav Bafna --- .../main/java/org/opensearch/cluster/SnapshotsInProgress.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 595f6d54fd9be..e3f98a3f61534 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -412,7 +412,7 @@ private Entry(StreamInput in) throws IOException { } else { remoteStoreIndexShallowCopy = false; } - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { remoteStoreIndexShallowCopyV2 = in.readBoolean(); } else { remoteStoreIndexShallowCopyV2 = false; @@ -886,7 +886,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_9_0)) { out.writeBoolean(remoteStoreIndexShallowCopy); } - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeBoolean(remoteStoreIndexShallowCopyV2); } } From 8ddb3eeee00978913fedd5aa49a84e718538ddae Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:15:15 +0530 Subject: [PATCH 162/357] Add success and failure count OTel metrics for async shard fetch (#15976) Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + .../TransportIndicesShardStoresAction.java | 27 ++- .../cluster/ClusterManagerMetrics.java | 13 + .../org/opensearch/cluster/ClusterModule.java | 3 + .../gateway/AsyncShardBatchFetch.java | 12 +- .../opensearch/gateway/AsyncShardFetch.java | 10 +- .../gateway/AsyncShardFetchCache.java | 7 +- .../opensearch/gateway/GatewayAllocator.java | 27 ++- .../gateway/ShardsBatchGatewayAllocator.java | 52 +++- .../gateway/AsyncShardFetchTests.java | 228 +++++++++++++++++- .../gateway/ShardBatchCacheTests.java | 5 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- 12 files changed, 344 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b628e9277959d..9a45bbaa879fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) +- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index a8b97d0f344ae..1a3c657f5b1b8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; @@ -88,6 +89,7 @@ public class TransportIndicesShardStoresAction extends TransportClusterManagerNo private static final Logger logger = LogManager.getLogger(TransportIndicesShardStoresAction.class); private final TransportNodesListGatewayStartedShards listShardStoresInfo; + private final ClusterManagerMetrics clusterManagerMetrics; @Inject public TransportIndicesShardStoresAction( @@ -96,7 +98,8 @@ public TransportIndicesShardStoresAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - TransportNodesListGatewayStartedShards listShardStoresInfo + TransportNodesListGatewayStartedShards listShardStoresInfo, + ClusterManagerMetrics clusterManagerMetrics ) { super( IndicesShardStoresAction.NAME, @@ -109,6 +112,7 @@ public TransportIndicesShardStoresAction( true ); this.listShardStoresInfo = listShardStoresInfo; + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -154,7 +158,7 @@ protected void clusterManagerOperation( // we could fetch all shard store info from every node once (nNodes requests) // we have to implement a TransportNodesAction instead of using TransportNodesListGatewayStartedShards // for fetching shard stores info, that operates on a list of shards instead of a single shard - new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener).start(); + new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener, clusterManagerMetrics).start(); } @Override @@ -175,12 +179,14 @@ private class AsyncShardStoresInfoFetches { private final ActionListener listener; private CountDown expectedOps; private final Queue fetchResponses; + private final ClusterManagerMetrics clusterManagerMetrics; AsyncShardStoresInfoFetches( DiscoveryNodes nodes, RoutingNodes routingNodes, Set> shards, - ActionListener listener + ActionListener listener, + ClusterManagerMetrics clusterManagerMetrics ) { this.nodes = nodes; this.routingNodes = routingNodes; @@ -188,6 +194,7 @@ private class AsyncShardStoresInfoFetches { this.listener = listener; this.fetchResponses = new ConcurrentLinkedQueue<>(); this.expectedOps = new CountDown(shards.size()); + this.clusterManagerMetrics = clusterManagerMetrics; } void start() { @@ -195,7 +202,14 @@ void start() { listener.onResponse(new IndicesShardStoresResponse()); } else { for (Tuple shard : shards) { - InternalAsyncFetch fetch = new InternalAsyncFetch(logger, "shard_stores", shard.v1(), shard.v2(), listShardStoresInfo); + InternalAsyncFetch fetch = new InternalAsyncFetch( + logger, + "shard_stores", + shard.v1(), + shard.v2(), + listShardStoresInfo, + clusterManagerMetrics + ); fetch.fetchData(nodes, Collections.emptyMap()); } } @@ -213,9 +227,10 @@ private class InternalAsyncFetch extends AsyncShardFetch deciderList; final ShardsAllocator shardsAllocator; + private final ClusterManagerMetrics clusterManagerMetrics; public ClusterModule( Settings settings, @@ -166,6 +167,7 @@ public ClusterModule( settings, clusterManagerMetrics ); + this.clusterManagerMetrics = clusterManagerMetrics; } public static List getNamedWriteables() { @@ -456,6 +458,7 @@ protected void configure() { bind(TaskResultsService.class).asEagerSingleton(); bind(AllocationDeciders.class).toInstance(allocationDeciders); bind(ShardsAllocator.class).toInstance(shardsAllocator); + bind(ClusterManagerMetrics.class).toInstance(clusterManagerMetrics); } public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator, ShardsBatchGatewayAllocator shardsBatchGatewayAllocator) { diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java index df642a9f5a743..d86d41bb1a359 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.logging.Loggers; import org.opensearch.core.index.shard.ShardId; @@ -48,7 +49,8 @@ public abstract class AsyncShardBatchFetch extend Class clazz, V emptyShardResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { super( logger, @@ -64,7 +66,8 @@ public abstract class AsyncShardBatchFetch extend clazz, emptyShardResponse, emptyShardResponsePredicate, - responseFactory + responseFactory, + clusterManagerMetrics ) ); } @@ -116,9 +119,10 @@ public ShardBatchCache( Class clazz, V emptyResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { - super(Loggers.getLogger(logger, "_" + logKey), type); + super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics); this.batchSize = shardAttributesMap.size(); this.emptyShardResponsePredicate = emptyShardResponsePredicate; cache = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java index b664dd573ce67..6017743ef2bd0 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java @@ -35,6 +35,7 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.RoutingAllocation; @@ -94,7 +95,8 @@ protected AsyncShardFetch( String type, ShardId shardId, String customDataPath, - Lister, T> action + Lister, T> action, + ClusterManagerMetrics clusterManagerMetrics ) { this.logger = logger; this.type = type; @@ -102,7 +104,7 @@ protected AsyncShardFetch( shardAttributesMap.put(shardId, new ShardAttributes(customDataPath)); this.action = (Lister, T>) action; this.reroutingKey = "ShardId=[" + shardId.toString() + "]"; - cache = new ShardCache<>(logger, reroutingKey, type); + cache = new ShardCache<>(logger, reroutingKey, type, clusterManagerMetrics); } /** @@ -284,8 +286,8 @@ static class ShardCache extends AsyncShardFetchCache private final Map> cache; - public ShardCache(Logger logger, String logKey, String type) { - super(Loggers.getLogger(logger, "_" + logKey), type); + public ShardCache(Logger logger, String logKey, String type, ClusterManagerMetrics clusterManagerMetrics) { + super(Loggers.getLogger(logger, "_" + logKey), type, clusterManagerMetrics); cache = new HashMap<>(); } diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java index 2a4e6181467b0..9b0a95f611e0e 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java @@ -14,6 +14,7 @@ import org.opensearch.OpenSearchTimeoutException; import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; @@ -51,10 +52,12 @@ public abstract class AsyncShardFetchCache { private final Logger logger; private final String type; + private final ClusterManagerMetrics clusterManagerMetrics; - protected AsyncShardFetchCache(Logger logger, String type) { + protected AsyncShardFetchCache(Logger logger, String type, ClusterManagerMetrics clusterManagerMetrics) { this.logger = logger; this.type = type; + this.clusterManagerMetrics = clusterManagerMetrics; } abstract void initData(DiscoveryNode node); @@ -162,6 +165,7 @@ Map getCacheData(DiscoveryNodes nodes, Set failedNodes } void processResponses(List responses, long fetchingRound) { + clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchSuccessCounter, Double.valueOf(responses.size())); for (K response : responses) { BaseNodeEntry nodeEntry = getCache().get(response.getNode().getId()); if (nodeEntry != null) { @@ -222,6 +226,7 @@ boolean retryableException(Throwable unwrappedCause) { } void processFailures(List failures, long fetchingRound) { + clusterManagerMetrics.incrementCounter(clusterManagerMetrics.asyncFetchFailureCounter, Double.valueOf(failures.size())); for (FailedNodeException failure : failures) { logger.trace("processing failure {} for [{}]", failure, type); BaseNodeEntry nodeEntry = getCache().get(failure.nodeId()); diff --git a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java index c8ef9364ebba9..eaacb5dbfbd17 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayAllocator.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -92,11 +93,12 @@ public class GatewayAllocator implements ExistingShardsAllocator { public GatewayAllocator( RerouteService rerouteService, TransportNodesListGatewayStartedShards startedAction, - TransportNodesListShardStoreMetadata storeAction + TransportNodesListShardStoreMetadata storeAction, + ClusterManagerMetrics clusterManagerMetrics ) { this.rerouteService = rerouteService; - this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction); - this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction); + this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction, clusterManagerMetrics); + this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction, clusterManagerMetrics); } @Override @@ -251,9 +253,10 @@ class InternalAsyncFetch extends AsyncShardFetch String type, ShardId shardId, String customDataPath, - Lister, T> action + Lister, T> action, + ClusterManagerMetrics clusterManagerMetrics ) { - super(logger, type, shardId, customDataPath, action); + super(logger, type, shardId, customDataPath, action, clusterManagerMetrics); } @Override @@ -274,9 +277,11 @@ protected void reroute(String reroutingKey, String reason) { class InternalPrimaryShardAllocator extends PrimaryShardAllocator { private final TransportNodesListGatewayStartedShards startedAction; + private final ClusterManagerMetrics clusterManagerMetrics; - InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction) { + InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction, ClusterManagerMetrics clusterManagerMetrics) { this.startedAction = startedAction; + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -291,7 +296,8 @@ protected AsyncShardFetch.FetchResult shardState = fetch.fetchData( @@ -313,9 +319,11 @@ protected AsyncShardFetch.FetchResult shardStores = fetch.fetchData( diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java index 5e2dcbcd70b40..d7c0a66ba3424 100644 --- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -44,6 +45,7 @@ import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata; import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper; import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper.StoreFilesMetadata; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import java.util.ArrayList; import java.util.Collections; @@ -81,6 +83,7 @@ public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator { private TimeValue primaryShardsBatchGatewayAllocatorTimeout; private TimeValue replicaShardsBatchGatewayAllocatorTimeout; public static final TimeValue MIN_ALLOCATOR_TIMEOUT = TimeValue.timeValueSeconds(20); + private final ClusterManagerMetrics clusterManagerMetrics; /** * Number of shards we send in one batch to data nodes for fetching metadata @@ -160,7 +163,8 @@ public ShardsBatchGatewayAllocator( TransportNodesListGatewayStartedShardsBatch batchStartedAction, TransportNodesListShardStoreMetadataBatch batchStoreAction, Settings settings, - ClusterSettings clusterSettings + ClusterSettings clusterSettings, + ClusterManagerMetrics clusterManagerMetrics ) { this.rerouteService = rerouteService; this.primaryShardBatchAllocator = new InternalPrimaryBatchShardAllocator(); @@ -172,6 +176,7 @@ public ShardsBatchGatewayAllocator( clusterSettings.addSettingsUpdateConsumer(PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setPrimaryBatchAllocatorTimeout); this.replicaShardsBatchGatewayAllocatorTimeout = REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setReplicaBatchAllocatorTimeout); + this.clusterManagerMetrics = clusterManagerMetrics; } @Override @@ -187,6 +192,7 @@ protected ShardsBatchGatewayAllocator() { this(DEFAULT_SHARD_BATCH_SIZE, null); } + // for tests protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteService) { this.rerouteService = rerouteService; this.batchStartedAction = null; @@ -196,10 +202,9 @@ protected ShardsBatchGatewayAllocator(long batchSize, RerouteService rerouteServ this.maxBatchSize = batchSize; this.primaryShardsBatchGatewayAllocatorTimeout = null; this.replicaShardsBatchGatewayAllocatorTimeout = null; + this.clusterManagerMetrics = new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE); } - // for tests - @Override public int getNumberOfInFlightFetches() { int count = 0; @@ -413,7 +418,7 @@ else if (shardRouting.primary() == primary) { // add to batch if batch size full or last shard in unassigned list if (batchSize == 0 || iterator.hasNext() == false) { String batchUUId = UUIDs.base64UUID(); - ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary); + ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary, clusterManagerMetrics); // add the batch to list of current batches addBatch(shardsBatch, primary); batchesToBeAssigned.add(batchUUId); @@ -588,9 +593,21 @@ class InternalBatchAsyncFetch extends AsyncShardB Class clazz, V emptyShardResponse, Predicate emptyShardResponsePredicate, - ShardBatchResponseFactory responseFactory + ShardBatchResponseFactory responseFactory, + ClusterManagerMetrics clusterManagerMetrics ) { - super(logger, type, map, action, batchUUId, clazz, emptyShardResponse, emptyShardResponsePredicate, responseFactory); + super( + logger, + type, + map, + action, + batchUUId, + clazz, + emptyShardResponse, + emptyShardResponsePredicate, + responseFactory, + clusterManagerMetrics + ); } @Override @@ -650,16 +667,17 @@ protected boolean hasInitiatedFetching(ShardRouting shard) { * It should return false if there has never been a fetch for this batch. * This function is currently only used in the case of replica shards when all deciders returned NO/THROTTLE, and explain mode is ON. * Allocation explain and manual reroute APIs try to append shard store information (matching bytes) to the allocation decision. - * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards, unless the data from nodes is already there. + * However, these APIs do not want to trigger a new asyncFetch for these ineligible shards + * They only want to use the data if it is already available. * This function is used to see if a fetch has happened to decide if it is possible to append shard store info without a new async fetch. * In the case when shard has a batch but no fetch has happened before, it would be because it is a new batch. * In the case when shard has a batch, and a fetch has happened before, and no fetch is ongoing, it would be because we have already completed fetch for all nodes. - * + *

* In order to check if a fetch has ever happened, we check 2 things: * 1. If the shard batch cache is empty, we know that fetch has never happened so we return false. * 2. If we see that the list of nodes to fetch from is empty, we know that all nodes have data or are ongoing a fetch. So we return true. * 3. Otherwise we return false. - * + *

* see {@link AsyncShardFetchCache#findNodesToFetch()} */ String batchId = getBatchId(shard, shard.primary()); @@ -669,7 +687,8 @@ protected boolean hasInitiatedFetching(ShardRouting shard) { logger.trace("Checking if fetching done for batch id {}", batchId); ShardsBatch shardsBatch = shard.primary() ? batchIdToStartedShardBatch.get(batchId) : batchIdToStoreShardBatch.get(batchId); // if fetchData has never been called, the per node cache will be empty and have no nodes - // this is because cache.fillShardCacheWithDataNodes(nodes) initialises this map and is called in AsyncShardFetch.fetchData + /// this is because {@link AsyncShardFetchCache#fillShardCacheWithDataNodes(DiscoveryNodes)} initialises this map + /// and is called in {@link AsyncShardFetch#fetchData(DiscoveryNodes, Map)} if (shardsBatch == null || shardsBatch.getAsyncFetcher().hasEmptyCache()) { logger.trace("Batch cache is empty for batch {} ", batchId); return false; @@ -739,7 +758,12 @@ public class ShardsBatch { private final Map batchInfo; - public ShardsBatch(String batchId, Map shardsWithInfo, boolean primary) { + public ShardsBatch( + String batchId, + Map shardsWithInfo, + boolean primary, + ClusterManagerMetrics clusterManagerMetrics + ) { this.batchId = batchId; this.batchInfo = new HashMap<>(shardsWithInfo); // create a ShardId -> customDataPath map for async fetch @@ -757,7 +781,8 @@ public ShardsBatch(String batchId, Map shardsWithInfo, bool GatewayStartedShard.class, new GatewayStartedShard(null, false, null, null), GatewayStartedShard::isEmpty, - new ShardBatchResponseFactory<>(true) + new ShardBatchResponseFactory<>(true), + clusterManagerMetrics ); } else { asyncBatch = new InternalBatchAsyncFetch<>( @@ -769,7 +794,8 @@ public ShardsBatch(String batchId, Map shardsWithInfo, bool NodeStoreFilesMetadata.class, new NodeStoreFilesMetadata(new StoreFilesMetadata(null, Store.MetadataSnapshot.EMPTY, Collections.emptyList()), null), NodeStoreFilesMetadata::isEmpty, - new ShardBatchResponseFactory<>(false) + new ShardBatchResponseFactory<>(false), + clusterManagerMetrics ); } } diff --git a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java index db97c3ece94ba..c25150873a1ce 100644 --- a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java +++ b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java @@ -35,10 +35,13 @@ import org.opensearch.Version; import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -54,6 +57,12 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.anyDouble; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class AsyncShardFetchTests extends OpenSearchTestCase { private final DiscoveryNode node1 = new DiscoveryNode( @@ -78,13 +87,29 @@ public class AsyncShardFetchTests extends OpenSearchTestCase { private ThreadPool threadPool; private TestFetch test; + private Counter asyncFetchSuccessCounter; + private Counter asyncFetchFailureCounter; + private Counter dummyCounter; @Override @Before public void setUp() throws Exception { super.setUp(); this.threadPool = new TestThreadPool(getTestName()); - this.test = new TestFetch(threadPool); + final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class); + this.asyncFetchFailureCounter = mock(Counter.class); + this.asyncFetchSuccessCounter = mock(Counter.class); + this.dummyCounter = mock(Counter.class); + when(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> { + String counterName = (String) invocationOnMock.getArguments()[0]; + if (counterName.contains("async.fetch.success.count")) { + return asyncFetchSuccessCounter; + } else if (counterName.contains("async.fetch.failure.count")) { + return asyncFetchFailureCounter; + } + return dummyCounter; + }); + this.test = new TestFetch(threadPool, metricsRegistry); } @After @@ -100,14 +125,26 @@ public void testClose() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter remains 0 because fetch is ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // counter goes up because fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(1)); test.close(); try { test.fetchData(nodes, emptyMap()); + // counter should not go up when calling fetchData since fetch never completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); fail("fetch data should fail when closed"); } catch (IllegalStateException e) { // all is well @@ -125,12 +162,21 @@ public void testFullCircleSingleNodeSuccess() throws Exception { // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // total counter goes up by 1 after success + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testFullCircleSingleNodeFailure() throws Exception { @@ -145,24 +191,47 @@ public void testFullCircleSingleNodeFailure() throws Exception { // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // failure, fetched data exists, but has no data assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(0)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // on failure, we reset the failure on a successive call to fetchData, and try again afterwards test.addSimulation(node1.getId(), response1); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // No additional failure, empty data so no change in counter + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); + // Success counter will increase + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // 2 reroutes, cause we have a failure that we clear assertThat(test.reroute.get(), equalTo(3)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter remains same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testIgnoreResponseFromDifferentRound() throws Exception { @@ -173,20 +242,40 @@ public void testIgnoreResponseFromDifferentRound() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetchData is not completed + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // handle a response with incorrect round id, wait on reroute incrementing test.processAsyncFetch(Collections.singletonList(response1), Collections.emptyList(), 0); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(1)); + // success counter increments to 1 because we called processAsyncFetch with a valid response, even though the round was incorrect + // failure counter also increments by 1 with empty list + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // fire a response (with correct round id), wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // success counter now goes up by 1 because fetchData completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); + // verify we get back the data node assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // total counter remains same because fetchdata does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(0.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testIgnoreFailureFromDifferentRound() throws Exception { @@ -198,6 +287,9 @@ public void testIgnoreFailureFromDifferentRound() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetchData still ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // handle a failure with incorrect round id, wait on reroute incrementing test.processAsyncFetch( @@ -207,14 +299,30 @@ public void testIgnoreFailureFromDifferentRound() throws Exception { ); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(1)); + // success counter called with empty list + // failure counter goes up by 1 because of the failure + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); // fire a response, wait on reroute incrementing test.fireSimulationAndWait(node1.getId()); + // failure counter goes up by 1 because of the failure + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(2)).add(1.0); + verify(asyncFetchFailureCounter, times(2)).add(anyDouble()); // failure, fetched data exists, but has no data assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(0)); + // counters remain same because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(0.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(2)).add(1.0); + verify(asyncFetchFailureCounter, times(2)).add(anyDouble()); } public void testTwoNodesOnSetup() throws Exception { @@ -226,16 +334,32 @@ public void testTwoNodesOnSetup() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); + // counter 1 because one fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); + // there is still another on going request, so no data assertThat(test.getNumberOfInFlightFetches(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // counter 2 because 2 fetches completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // no more ongoing requests, we should fetch the data assertThat(test.reroute.get(), equalTo(2)); fetchData = test.fetchData(nodes, emptyMap()); @@ -243,6 +367,10 @@ public void testTwoNodesOnSetup() throws Exception { assertThat(fetchData.getData().size(), equalTo(2)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); assertThat(fetchData.getData().get(node2), sameInstance(response2)); + // counter still 2 because fetchData call did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testTwoNodesOnSetupAndFailure() throws Exception { @@ -254,34 +382,59 @@ public void testTwoNodesOnSetupAndFailure() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because both fetches ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter 1 because one fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // failure counter up by 1 because one fetch failed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); assertThat(test.reroute.get(), equalTo(2)); + // since one of those failed, we should only have one entry fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // success and failure counters same because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(1)).add(1.0); + verify(asyncFetchFailureCounter, times(1)).add(anyDouble()); } public void testTwoNodesAddedInBetween() throws Exception { DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).build(); test.addSimulation(node1.getId(), response1); - // no fetched data, 2 requests still on going + // no fetched data, request still on going AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because both fetches ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the first response, it should trigger a reroute test.fireSimulationAndWait(node1.getId()); + // counter 1 because fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // now, add a second node to the nodes, it should add it to the ongoing requests nodes = DiscoveryNodes.builder(nodes).add(node2).build(); @@ -289,16 +442,28 @@ public void testTwoNodesAddedInBetween() throws Exception { // no fetch data, has a new node introduced fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because second fetch ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // fire the second simulation, this should allow us to get the data test.fireSimulationAndWait(node2.getId()); + // counter now 2 because 2 fetches completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); - // since one of those failed, we should only have one entry + // since both succeeded, we should have 2 entries fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(2)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); assertThat(fetchData.getData().get(node2), sameInstance(response2)); + // counter still 2 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testClearCache() throws Exception { @@ -312,21 +477,36 @@ public void testClearCache() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); + // counter 1 because 1 fetch completed + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get back right data from node fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter still 1 because a new fetch is not called + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // second fetch gets same data fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1)); + // counter still 1 because a new fetch is not called + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.clearCacheForNode(node1.getId()); @@ -336,15 +516,27 @@ public void testClearCache() throws Exception { // no fetched data, new request on going fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter still 1 because new fetch is still ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(2)); + // counter now 2 because second fetch completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get new data back fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1_2)); + // counter still 2 because fetchData did not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } public void testConcurrentRequestAndClearCache() throws Exception { @@ -355,12 +547,19 @@ public void testConcurrentRequestAndClearCache() throws Exception { AsyncShardFetch.FetchResult fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); assertThat(test.reroute.get(), equalTo(0)); + // counter 0 because fetch ongoing + verify(asyncFetchSuccessCounter, times(0)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // clear cache while request is still on going, before it is processed test.clearCacheForNode(node1.getId()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(1)); + // counter 1 because fetch completed, even though cache was wiped + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // prepare next request test.addSimulation(node1.getId(), response1_2); @@ -368,15 +567,27 @@ public void testConcurrentRequestAndClearCache() throws Exception { // verify still no fetched data, request still on going fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(false)); + // counter unchanged because fetch ongoing + verify(asyncFetchSuccessCounter, times(1)).add(1.0); + verify(asyncFetchSuccessCounter, times(1)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); test.fireSimulationAndWait(node1.getId()); assertThat(test.reroute.get(), equalTo(2)); + // counter 2 because second fetch completed + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); // verify we get new data back fetchData = test.fetchData(nodes, emptyMap()); assertThat(fetchData.hasData(), equalTo(true)); assertThat(fetchData.getData().size(), equalTo(1)); assertThat(fetchData.getData().get(node1), sameInstance(response1_2)); + // counter unchanged because fetchData does not trigger new async fetch + verify(asyncFetchSuccessCounter, times(2)).add(1.0); + verify(asyncFetchSuccessCounter, times(2)).add(anyDouble()); + verify(asyncFetchFailureCounter, times(0)).add(anyDouble()); } @@ -398,8 +609,15 @@ static class Entry { private final Map simulations = new ConcurrentHashMap<>(); private AtomicInteger reroute = new AtomicInteger(); - TestFetch(ThreadPool threadPool) { - super(LogManager.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), "", null); + TestFetch(ThreadPool threadPool, MetricsRegistry metricsRegistry) { + super( + LogManager.getLogger(TestFetch.class), + "test", + new ShardId("test", "_na_", 1), + "", + null, + new ClusterManagerMetrics(metricsRegistry) + ); this.threadPool = threadPool; } diff --git a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java index 12030ad41d508..39c4ee8c8ca06 100644 --- a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java +++ b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java @@ -8,6 +8,7 @@ package org.opensearch.gateway; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.OpenSearchAllocationTestCase; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -19,6 +20,7 @@ import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard; import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch; import org.opensearch.indices.store.ShardAttributes; +import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import java.util.ArrayList; import java.util.HashMap; @@ -52,7 +54,8 @@ public void setupShardBatchCache(String batchId, int numberOfShards) { GatewayStartedShard.class, new GatewayStartedShard(null, false, null, null), GatewayStartedShard::isEmpty, - new ShardBatchResponseFactory<>(true) + new ShardBatchResponseFactory<>(true), + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 440227436175d..d17e661615b0d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2429,7 +2429,8 @@ public void onFailure(final Exception e) { nodeEnv, indicesService, namedXContentRegistry - ) + ), + new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE) ) ); actions.put( From e6e290d7d77feec1404e296d130aff536c482392 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 30 Sep 2024 19:41:12 +0800 Subject: [PATCH 163/357] Fix version check in yml test file for double range profiler shows filter rewrite info (#16125) Signed-off-by: Gao Binlong --- .../rest-api-spec/test/search.aggregation/40_range.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index 1e1d2b0706d6b..93ca668f93a6f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -535,8 +535,8 @@ setup: --- "Double range profiler shows filter rewrite info": - skip: - version: " - 2.99.99" - reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.15.0) + version: " - 2.15.99" + reason: debug info for filter rewrite was added in 2.16.0 - do: indices.create: From 0b96565b51326e916e9fe33ead44187844c23c46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:13:34 -0400 Subject: [PATCH 164/357] Bump com.maxmind.db:maxmind-db from 3.1.0 to 3.1.1 in /modules/ingest-geoip (#16137) * Bump com.maxmind.db:maxmind-db in /modules/ingest-geoip Bumps [com.maxmind.db:maxmind-db](https://github.com/maxmind/MaxMind-DB-Reader-java) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/maxmind/MaxMind-DB-Reader-java/releases) - [Changelog](https://github.com/maxmind/MaxMind-DB-Reader-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/maxmind/MaxMind-DB-Reader-java/compare/v3.1.0...v3.1.1) --- updated-dependencies: - dependency-name: com.maxmind.db:maxmind-db dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + modules/ingest-geoip/build.gradle | 2 +- modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 | 1 - modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a45bbaa879fd..b01badbe3fd2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) +- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 162ac6f9ad620..f74de1dc290dd 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -41,7 +41,7 @@ opensearchplugin { dependencies { api('com.maxmind.geoip2:geoip2:4.2.1') // geoip2 dependencies: - api('com.maxmind.db:maxmind-db:3.1.0') + api('com.maxmind.db:maxmind-db:3.1.1') api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 deleted file mode 100644 index 9db7c7319af0b..0000000000000 --- a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2008992ab45d61c7b28a18678b5df82272529da3 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 new file mode 100644 index 0000000000000..f1a25c6d600c0 --- /dev/null +++ b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 @@ -0,0 +1 @@ +84a2a5d322fef1f1e82f05095683c8d53902baf1 \ No newline at end of file From 9a5c7b2eae405d08c4d5d64f0fef087821c18655 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 30 Sep 2024 22:16:52 +0530 Subject: [PATCH 165/357] Add default implementation to new finalizeSnapshot() in Repository (#16128) Signed-off-by: Sachin Kale --- .../main/java/org/opensearch/repositories/Repository.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index d4520beb5b570..138bc13140aea 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -180,7 +180,7 @@ void finalizeSnapshot( * @param repositoryUpdatePriority priority for the cluster state update task * @param listener listener to be invoked with the new {@link RepositoryData} after completing the snapshot */ - void finalizeSnapshot( + default void finalizeSnapshot( ShardGenerations shardGenerations, long repositoryStateId, Metadata clusterMetadata, @@ -189,7 +189,9 @@ void finalizeSnapshot( Function stateTransformer, Priority repositoryUpdatePriority, ActionListener listener - ); + ) { + throw new UnsupportedOperationException(); + } /** * Deletes snapshots From d131d58d842d18e62e523d5c6833d8c86652ce81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:24:41 -0400 Subject: [PATCH 166/357] Bump org.apache.logging.log4j:log4j-core from 2.24.0 to 2.24.1 in /buildSrc/src/testKit/thirdPartyAudit/sample_jars (#16134) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.24.0 to 2.24.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b01badbe3fd2e..da14948781849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) -- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 59063b3789c70..4d425964c77af 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.24.0" + implementation "org.apache.logging.log4j:log4j-core:2.24.1" } ["0.0.1", "0.0.2"].forEach { v -> From 210228fcbe5c6d5efad417de64ae01c83927ed87 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 30 Sep 2024 21:11:50 -0400 Subject: [PATCH 167/357] Add 2.17.1 release notes (#16104) (#16105) (#16132) (cherry picked from commit dff2184279d7392f44cb52888916bdc9020a5e61) (cherry picked from commit af105b1bb547fc8843fab0986f1aa60454529d54) Signed-off-by: Andriy Redko --- release-notes/opensearch.release-notes-2.17.1.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 release-notes/opensearch.release-notes-2.17.1.md diff --git a/release-notes/opensearch.release-notes-2.17.1.md b/release-notes/opensearch.release-notes-2.17.1.md new file mode 100644 index 0000000000000..2ff2bd44c3b1c --- /dev/null +++ b/release-notes/opensearch.release-notes-2.17.1.md @@ -0,0 +1,16 @@ +## 2024-10-01 Version 2.17.1 Release Notes + +## [2.17.1] +### Added +- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664)) +- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492)) + +### Dependencies + +### Changed + +### Deprecated + +### Removed + +### Fixed From 43e7597cdc5ba2c1852ec1796628f948633f0c57 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:13:15 +0530 Subject: [PATCH 168/357] Add assertBusy to failing snapshot tests (#16146) Signed-off-by: Lakshya Taragi --- .../snapshots/SnapshotStatusApisIT.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index c3214022df663..8b6869aa1d81a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -116,7 +116,7 @@ public void testStatusApiConsistency() { assertEquals(snapshotStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime()); } - public void testStatusAPICallForShallowCopySnapshot() { + public void testStatusAPICallForShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNode(); @@ -136,15 +136,24 @@ public void testStatusAPICallForShallowCopySnapshot() { final String snapshot = "snapshot"; createFullSnapshot(snapshotRepoName, snapshot); - final SnapshotStatus snapshotStatus = getSnapshotStatus(snapshotRepoName, snapshot); - assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); + assertBusy(() -> { + final SnapshotStatus snapshotStatus = client().admin() + .cluster() + .prepareSnapshotStatus(snapshotRepoName) + .setSnapshots(snapshot) + .execute() + .actionGet() + .getSnapshots() + .get(0); + assertThat(snapshotStatus.getState(), is(SnapshotsInProgress.State.SUCCESS)); - final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName); - assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); - assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); - assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); - assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); - assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); + final SnapshotIndexShardStatus snapshotShardState = stateFirstShard(snapshotStatus, indexName); + assertThat(snapshotShardState.getStage(), is(SnapshotIndexShardStage.DONE)); + assertThat(snapshotShardState.getStats().getTotalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getTotalSize(), greaterThan(0L)); + assertThat(snapshotShardState.getStats().getIncrementalFileCount(), greaterThan(0)); + assertThat(snapshotShardState.getStats().getIncrementalSize(), greaterThan(0L)); + }, 20, TimeUnit.SECONDS); } public void testStatusAPICallInProgressSnapshot() throws Exception { @@ -193,7 +202,7 @@ public void testExceptionOnMissingSnapBlob() throws IOException { ); } - public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { + public void testExceptionOnMissingShardLevelSnapBlob() throws Exception { disableRepoConsistencyCheck("This test intentionally corrupts the repository"); final Path repoPath = randomRepoPath(); @@ -216,11 +225,12 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { repoPath.resolve(resolvePath(indexId, "0")) .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat") ); - - expectThrows( - SnapshotMissingException.class, - () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet() - ); + assertBusy(() -> { + expectThrows( + SnapshotMissingException.class, + () -> client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap").execute().actionGet() + ); + }, 20, TimeUnit.SECONDS); } public void testGetSnapshotsWithoutIndices() throws Exception { From a767e92f3eeaf57c066deb9ad075d40ed00f4a58 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 1 Oct 2024 14:48:10 +0530 Subject: [PATCH 169/357] Optimize checksum creation for remote cluster state (#16046) * Support parallelisation in remote publication checksum computation Signed-off-by: Himshikha Gupta --- .../gateway/remote/ClusterStateChecksum.java | 150 ++++++++++++------ .../remote/RemoteClusterStateService.java | 14 +- .../org/opensearch/threadpool/ThreadPool.java | 7 + .../remote/ClusterMetadataManifestTests.java | 15 +- .../remote/ClusterStateChecksumTests.java | 32 ++-- .../RemoteClusterStateServiceTests.java | 28 ++-- 6 files changed, 164 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java index d6739c4572d1a..aa007f5da15b3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateChecksum.java @@ -12,8 +12,10 @@ import org.apache.logging.log4j.Logger; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.CheckedFunction; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -22,11 +24,15 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import com.jcraft.jzlib.JZlib; @@ -37,6 +43,7 @@ */ public class ClusterStateChecksum implements ToXContentFragment, Writeable { + public static final int COMPONENT_SIZE = 11; static final String ROUTING_TABLE_CS = "routing_table"; static final String NODES_CS = "discovery_nodes"; static final String BLOCKS_CS = "blocks"; @@ -65,62 +72,103 @@ public class ClusterStateChecksum implements ToXContentFragment, Writeable { long indicesChecksum; long clusterStateChecksum; - public ClusterStateChecksum(ClusterState clusterState) { - try ( - BytesStreamOutput out = new BytesStreamOutput(); - BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) - ) { - clusterState.routingTable().writeVerifiableTo(checksumOut); - routingTableChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.nodes().writeVerifiableTo(checksumOut); - nodesChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.coordinationMetadata().writeVerifiableTo(checksumOut); - coordinationMetadataChecksum = checksumOut.getChecksum(); - - // Settings create sortedMap by default, so no explicit sorting required here. - checksumOut.reset(); - Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), checksumOut); - settingMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), checksumOut); - transientSettingsMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.metadata().templatesMetadata().writeVerifiableTo(checksumOut); - templatesMetadataChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeStringCollection(clusterState.metadata().customs().keySet()); - customMetadataMapChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(checksumOut); - hashesOfConsistentSettingsChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeMapValues( + public ClusterStateChecksum(ClusterState clusterState, ThreadPool threadpool) { + long start = threadpool.relativeTimeInNanos(); + ExecutorService executorService = threadpool.executor(ThreadPool.Names.REMOTE_STATE_CHECKSUM); + CountDownLatch latch = new CountDownLatch(COMPONENT_SIZE); + + executeChecksumTask((stream) -> { + clusterState.routingTable().writeVerifiableTo(stream); + return null; + }, checksum -> routingTableChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.nodes().writeVerifiableTo(stream); + return null; + }, checksum -> nodesChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.coordinationMetadata().writeVerifiableTo(stream); + return null; + }, checksum -> coordinationMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + Settings.writeSettingsToStream(clusterState.metadata().persistentSettings(), stream); + return null; + }, checksum -> settingMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + Settings.writeSettingsToStream(clusterState.metadata().transientSettings(), stream); + return null; + }, checksum -> transientSettingsMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.metadata().templatesMetadata().writeVerifiableTo(stream); + return null; + }, checksum -> templatesMetadataChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeStringCollection(clusterState.metadata().customs().keySet()); + return null; + }, checksum -> customMetadataMapChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + ((DiffableStringMap) clusterState.metadata().hashesOfConsistentSettings()).writeTo(stream); + return null; + }, checksum -> hashesOfConsistentSettingsChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeMapValues( clusterState.metadata().indices(), - (stream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) stream) + (checksumStream, value) -> value.writeVerifiableTo((BufferedChecksumStreamOutput) checksumStream) ); - indicesChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - clusterState.blocks().writeVerifiableTo(checksumOut); - blocksChecksum = checksumOut.getChecksum(); - - checksumOut.reset(); - checksumOut.writeStringCollection(clusterState.customs().keySet()); - clusterStateCustomsChecksum = checksumOut.getChecksum(); - } catch (IOException e) { - logger.error("Failed to create checksum for cluster state.", e); + return null; + }, checksum -> indicesChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + clusterState.blocks().writeVerifiableTo(stream); + return null; + }, checksum -> blocksChecksum = checksum, executorService, latch); + + executeChecksumTask((stream) -> { + stream.writeStringCollection(clusterState.customs().keySet()); + return null; + }, checksum -> clusterStateCustomsChecksum = checksum, executorService, latch); + + try { + latch.await(); + } catch (InterruptedException e) { throw new RemoteStateTransferException("Failed to create checksum for cluster state.", e); } createClusterStateChecksum(); + logger.debug("Checksum execution time {}", TimeValue.nsecToMSec(threadpool.relativeTimeInNanos() - start)); + } + + private void executeChecksumTask( + CheckedFunction checksumTask, + Consumer checksumConsumer, + ExecutorService executorService, + CountDownLatch latch + ) { + executorService.execute(() -> { + try { + long checksum = createChecksum(checksumTask); + checksumConsumer.accept(checksum); + latch.countDown(); + } catch (IOException e) { + throw new RemoteStateTransferException("Failed to execute checksum task", e); + } + }); + } + + private long createChecksum(CheckedFunction task) throws IOException { + try ( + BytesStreamOutput out = new BytesStreamOutput(); + BufferedChecksumStreamOutput checksumOut = new BufferedChecksumStreamOutput(out) + ) { + task.apply(checksumOut); + return checksumOut.getChecksum(); + } } private void createClusterStateChecksum() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ece29180f9cf5..ce5e57b79dadb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -332,7 +332,9 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat uploadedMetadataResults, previousClusterUUID, clusterStateDiffManifest, - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, false, codecVersion ); @@ -539,7 +541,9 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), clusterStateDiffManifest, - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, false, previousManifest.getCodecVersion() ); @@ -1010,7 +1014,9 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted( uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), - !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState) : null, + !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + ? new ClusterStateChecksum(clusterState, threadpool) + : null, true, previousManifest.getCodecVersion() ); @@ -1631,7 +1637,7 @@ void validateClusterStateFromChecksum( String localNodeId, boolean isFullStateDownload ) { - ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState); + ClusterStateChecksum newClusterStateChecksum = new ClusterStateChecksum(clusterState, threadpool); List failedValidation = newClusterStateChecksum.getMismatchEntities(manifest.getClusterStateChecksum()); if (failedValidation.isEmpty()) { return; diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index 81220ab171b34..d795fd252b7fc 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -53,6 +53,7 @@ import org.opensearch.core.service.ReportingService; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.gateway.remote.ClusterStateChecksum; import org.opensearch.node.Node; import java.io.IOException; @@ -118,6 +119,7 @@ public static class Names { public static final String REMOTE_RECOVERY = "remote_recovery"; public static final String REMOTE_STATE_READ = "remote_state_read"; public static final String INDEX_SEARCHER = "index_searcher"; + public static final String REMOTE_STATE_CHECKSUM = "remote_state_checksum"; } /** @@ -191,6 +193,7 @@ public static ThreadPoolType fromType(String type) { map.put(Names.REMOTE_RECOVERY, ThreadPoolType.SCALING); map.put(Names.REMOTE_STATE_READ, ThreadPoolType.SCALING); map.put(Names.INDEX_SEARCHER, ThreadPoolType.RESIZABLE); + map.put(Names.REMOTE_STATE_CHECKSUM, ThreadPoolType.FIXED); THREAD_POOL_TYPES = Collections.unmodifiableMap(map); } @@ -307,6 +310,10 @@ public ThreadPool( runnableTaskListener ) ); + builders.put( + Names.REMOTE_STATE_CHECKSUM, + new FixedExecutorBuilder(settings, Names.REMOTE_STATE_CHECKSUM, ClusterStateChecksum.COMPONENT_SIZE, 1000) + ); for (final ExecutorBuilder builder : customBuilders) { if (builders.containsKey(builder.name())) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 3f9aa1245cab3..09c2933680be3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -34,6 +34,9 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; import java.io.IOException; import java.util.ArrayList; @@ -64,6 +67,14 @@ public class ClusterMetadataManifestTests extends OpenSearchTestCase { + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @After + public void teardown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + public void testClusterMetadataManifestXContentV0() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path", CODEC_V0); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() @@ -214,7 +225,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "indicesRoutingDiffPath" ) ) - .checksum(new ClusterStateChecksum(createClusterState())) + .checksum(new ClusterStateChecksum(createClusterState(), threadPool)) .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -647,7 +658,7 @@ public void testClusterMetadataManifestXContentV4() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); final StringKeyDiffProvider routingTableIncrementalDiff = Mockito.mock(StringKeyDiffProvider.class); - ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(createClusterState(), threadPool); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java index 0203e56dd2d5c..9b98187053a39 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterStateChecksumTests.java @@ -34,6 +34,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; import java.io.IOException; import java.util.EnumSet; @@ -41,14 +44,21 @@ import java.util.Map; public class ClusterStateChecksumTests extends OpenSearchTestCase { + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @After + public void teardown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } public void testClusterStateChecksumEmptyClusterState() { - ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE); + ClusterStateChecksum checksum = new ClusterStateChecksum(ClusterState.EMPTY_STATE, threadPool); assertNotNull(checksum); } public void testClusterStateChecksum() { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); assertNotNull(checksum); assertTrue(checksum.routingTableChecksum != 0); assertTrue(checksum.nodesChecksum != 0); @@ -65,8 +75,8 @@ public void testClusterStateChecksum() { } public void testClusterStateMatchChecksum() { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); - ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); + ClusterStateChecksum newChecksum = new ClusterStateChecksum(generateClusterState(), threadPool); assertNotNull(checksum); assertNotNull(newChecksum); assertEquals(checksum.routingTableChecksum, newChecksum.routingTableChecksum); @@ -84,7 +94,7 @@ public void testClusterStateMatchChecksum() { } public void testXContentConversion() throws IOException { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); checksum.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -97,7 +107,7 @@ public void testXContentConversion() throws IOException { } public void testSerialization() throws IOException { - ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState()); + ClusterStateChecksum checksum = new ClusterStateChecksum(generateClusterState(), threadPool); BytesStreamOutput output = new BytesStreamOutput(); checksum.writeTo(output); @@ -109,10 +119,10 @@ public void testSerialization() throws IOException { public void testGetMismatchEntities() { ClusterState clsState1 = generateClusterState(); - ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1); + ClusterStateChecksum checksum = new ClusterStateChecksum(clsState1, threadPool); assertTrue(checksum.getMismatchEntities(checksum).isEmpty()); - ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(clsState1, threadPool); assertTrue(checksum.getMismatchEntities(checksum2).isEmpty()); ClusterState clsState2 = ClusterState.builder(ClusterName.DEFAULT) @@ -122,7 +132,7 @@ public void testGetMismatchEntities() { .customs(Map.of()) .metadata(Metadata.EMPTY_METADATA) .build(); - ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2); + ClusterStateChecksum checksum3 = new ClusterStateChecksum(clsState2, threadPool); List mismatches = checksum.getMismatchEntities(checksum3); assertFalse(mismatches.isEmpty()); assertEquals(11, mismatches.size()); @@ -151,8 +161,8 @@ public void testGetMismatchEntitiesUnorderedInput() { ClusterState state2 = ClusterState.builder(state1).nodes(nodes1).build(); ClusterState state3 = ClusterState.builder(state1).nodes(nodes2).build(); - ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2); - ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3); + ClusterStateChecksum checksum1 = new ClusterStateChecksum(state2, threadPool); + ClusterStateChecksum checksum2 = new ClusterStateChecksum(state3, threadPool); assertEquals(checksum2, checksum1); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 56857285fa8d3..35a8ae16cacf7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -3123,7 +3123,7 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1L) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3193,7 +3193,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indices(Collections.emptyList()) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -3219,7 +3219,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationEnabled() t .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3245,7 +3245,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone() final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() .indices(Collections.emptyList()) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); @@ -3271,7 +3271,7 @@ public void testWriteIncrementalMetadataSuccessWithChecksumValidationModeNone() .previousClusterUUID("prev-cluster-uuid") .routingTableVersion(1) .indicesRouting(List.of(uploadedIndiceRoutingMetadata)) - .checksum(new ClusterStateChecksum(clusterState)) + .checksum(new ClusterStateChecksum(clusterState, threadPool)) .build(); assertThat(manifest.getIndices().size(), is(1)); @@ -3349,7 +3349,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabled() throws initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3382,7 +3382,7 @@ public void testGetClusterStateForManifestWithChecksumValidationModeNone() throw initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3415,7 +3415,7 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3465,7 +3465,7 @@ public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatc ); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).build(); remoteClusterStateService.start(); RemoteClusterStateService mockService = spy(remoteClusterStateService); @@ -3505,7 +3505,7 @@ public void testGetClusterStateUsingDiffWithChecksum() throws IOException { initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3547,7 +3547,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeNone() throws IOExceptio initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.NONE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3589,7 +3589,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeDebugMismatch() throws I initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.DEBUG); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3630,7 +3630,7 @@ public void testGetClusterStateUsingDiffWithChecksumModeTraceMismatch() throws I initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.TRACE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); @@ -3692,7 +3692,7 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio initializeWithChecksumEnabled(RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE); ClusterState clusterState = generateClusterStateWithAllAttributes().build(); ClusterMetadataManifest manifest = generateClusterMetadataManifestWithAllAttributes().checksum( - new ClusterStateChecksum(clusterState) + new ClusterStateChecksum(clusterState, threadPool) ).diffManifest(ClusterStateDiffManifest.builder().build()).build(); remoteClusterStateService.start(); From 1ef6444cfa1ab805ba5d68a3078274cbbb2b9a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 09:14:28 -0400 Subject: [PATCH 170/357] Bump com.azure:azure-core-http-netty from 1.15.3 to 1.15.4 in /plugins/repository-azure (#16133) * Bump com.azure:azure-core-http-netty in /plugins/repository-azure Bumps [com.azure:azure-core-http-netty](https://github.com/Azure/azure-sdk-for-java) from 1.15.3 to 1.15.4. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core-http-netty_1.15.3...azure-core-http-netty_1.15.4) --- updated-dependencies: - dependency-name: com.azure:azure-core-http-netty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-core-http-netty-1.15.3.jar.sha1 | 1 - .../licenses/azure-core-http-netty-1.15.4.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index da14948781849..038cc407d582b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) +- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 2892bdba51ba6..4baf79e619be9 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -48,7 +48,7 @@ dependencies { api 'com.azure:azure-json:1.1.0' api 'com.azure:azure-xml:1.1.0' api 'com.azure:azure-storage-common:12.25.1' - api 'com.azure:azure-core-http-netty:1.15.3' + api 'com.azure:azure-core-http-netty:1.15.4' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" api "io.netty:netty-codec-http2:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 deleted file mode 100644 index 3cea52ba67ce5..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03b5bd5f5c16eea71f130119dbfb1fe5239f806a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 new file mode 100644 index 0000000000000..97e6fad264294 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 @@ -0,0 +1 @@ +489a38c9e6efb5ce01fbd276d8cb6c0e89000459 \ No newline at end of file From be9f94258315405aee203a0737decd4e19343942 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Tue, 1 Oct 2024 19:07:29 +0530 Subject: [PATCH 171/357] [SnapshotV2] Add timestamp of last successful fetch of pinned timestamps in node stats (#15611) --------- Signed-off-by: Lakshya Taragi --- CHANGELOG.md | 1 + .../RemoteStorePinnedTimestampsIT.java | 41 ++++++++++ .../admin/cluster/node/stats/NodeStats.java | 26 +++++- .../cluster/node/stats/NodesStatsRequest.java | 3 +- .../node/stats/TransportNodesStatsAction.java | 3 +- .../stats/TransportClusterStatsAction.java | 1 + .../java/org/opensearch/node/NodeService.java | 7 +- .../remotestore/RemoteStoreNodeStats.java | 79 +++++++++++++++++++ .../cluster/node/stats/NodeStatsTests.java | 22 +++++- .../cluster/stats/ClusterStatsNodesTests.java | 1 + .../opensearch/cluster/DiskUsageTests.java | 6 ++ .../MockInternalClusterInfoService.java | 3 +- .../opensearch/test/InternalTestCluster.java | 1 + 13 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 038cc407d582b..1a0859a7d5af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) +- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index 2fcda8c2d2f27..024e0e952eea5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -9,6 +9,8 @@ package org.opensearch.remotestore; import org.opensearch.action.LatchedActionListener; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -20,6 +22,8 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; +import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.REMOTE_STORE; + @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase { static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -180,4 +184,41 @@ public void onFailure(Exception e) { assertBusy(() -> assertEquals(Set.of(timestamp2, timestamp3), RemoteStorePinnedTimestampService.getPinnedTimestamps().v2())); remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); } + + public void testLastSuccessfulFetchOfPinnedTimestampsPresentInNodeStats() throws Exception { + logger.info("Starting up cluster manager"); + logger.info("cluster.remote_store.pinned_timestamps.enabled set to true"); + logger.info("cluster.remote_store.pinned_timestamps.scheduler_interval set to minimum value of 1minute"); + Settings pinnedTimestampEnabledSettings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_SCHEDULER_INTERVAL.getKey(), "1m") + .build(); + internalCluster().startClusterManagerOnlyNode(pinnedTimestampEnabledSettings); + String remoteNodeName = internalCluster().startDataOnlyNodes(1, pinnedTimestampEnabledSettings).get(0); + ensureStableCluster(2); + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + remoteNodeName + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + + assertBusy(() -> { + long lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1(); + assertTrue(lastSuccessfulFetchOfPinnedTimestamps > 0L); + NodesStatsResponse nodesStatsResponse = internalCluster().client() + .admin() + .cluster() + .prepareNodesStats() + .addMetric(REMOTE_STORE.metricName()) + .execute() + .actionGet(); + for (NodeStats nodeStats : nodesStatsResponse.getNodes()) { + long lastRecordedFetch = nodeStats.getRemoteStoreNodeStats().getLastSuccessfulFetchOfPinnedTimestamps(); + assertTrue(lastRecordedFetch >= lastSuccessfulFetchOfPinnedTimestamps); + } + }); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 0917a0baff1ab..c91260778f037 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -59,6 +59,7 @@ import org.opensearch.monitor.process.ProcessStats; import org.opensearch.node.AdaptiveSelectionStats; import org.opensearch.node.NodesResourceUsageStats; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.ratelimitting.admissioncontrol.stats.AdmissionControlStats; import org.opensearch.repositories.RepositoriesStats; import org.opensearch.script.ScriptCacheStats; @@ -162,6 +163,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private NodeCacheStats nodeCacheStats; + @Nullable + private RemoteStoreNodeStats remoteStoreNodeStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -243,6 +247,12 @@ public NodeStats(StreamInput in) throws IOException { } else { nodeCacheStats = null; } + // TODO: change version to V_2_18_0 + if (in.getVersion().onOrAfter(Version.CURRENT)) { + remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new); + } else { + remoteStoreNodeStats = null; + } } public NodeStats( @@ -274,7 +284,8 @@ public NodeStats( @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats, @Nullable RepositoriesStats repositoriesStats, @Nullable AdmissionControlStats admissionControlStats, - @Nullable NodeCacheStats nodeCacheStats + @Nullable NodeCacheStats nodeCacheStats, + @Nullable RemoteStoreNodeStats remoteStoreNodeStats ) { super(node); this.timestamp = timestamp; @@ -305,6 +316,7 @@ public NodeStats( this.repositoriesStats = repositoriesStats; this.admissionControlStats = admissionControlStats; this.nodeCacheStats = nodeCacheStats; + this.remoteStoreNodeStats = remoteStoreNodeStats; } public long getTimestamp() { @@ -467,6 +479,11 @@ public NodeCacheStats getNodeCacheStats() { return nodeCacheStats; } + @Nullable + public RemoteStoreNodeStats getRemoteStoreNodeStats() { + return remoteStoreNodeStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -525,6 +542,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeOptionalWriteable(nodeCacheStats); } + // TODO: change version to V_2_18_0 + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalWriteable(remoteStoreNodeStats); + } } @Override @@ -631,6 +652,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getNodeCacheStats() != null) { getNodeCacheStats().toXContent(builder, params); } + if (getRemoteStoreNodeStats() != null) { + getRemoteStoreNodeStats().toXContent(builder, params); + } return builder; } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index f1f9f93afdad2..a5b00ed82d3cb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -220,7 +220,8 @@ public enum Metric { SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"), REPOSITORIES("repositories"), ADMISSION_CONTROL("admission_control"), - CACHE_STATS("caches"); + CACHE_STATS("caches"), + REMOTE_STORE("remote_store"); private String metricName; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 2c808adc97c7a..a98d245af872b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -129,7 +129,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) { NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics), NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics), NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics), - NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics) + NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics), + NodesStatsRequest.Metric.REMOTE_STORE.containedIn(metrics) ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index a49ca2035783c..c4b3524cf6da5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -174,6 +174,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false, false, + false, false ); List shardsStats = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 1eb38ea63ad5a..9671fda14375d 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -54,6 +54,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.ingest.IngestService; import org.opensearch.monitor.MonitorService; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.plugins.PluginsService; import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService; import org.opensearch.repositories.RepositoriesService; @@ -241,7 +242,8 @@ public NodeStats stats( boolean segmentReplicationTrackerStats, boolean repositoriesStats, boolean admissionControl, - boolean cacheService + boolean cacheService, + boolean remoteStoreNodeStats ) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) @@ -274,7 +276,8 @@ public NodeStats stats( segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null, repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null, admissionControl ? this.admissionControlService.stats() : null, - cacheService ? this.cacheService.stats(indices) : null + cacheService ? this.cacheService.stats(indices) : null, + remoteStoreNodeStats ? new RemoteStoreNodeStats() : null ); } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java new file mode 100644 index 0000000000000..8da8a17e21839 --- /dev/null +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeStats.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.node.remotestore; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Node level remote store stats + * @opensearch.internal + */ +public class RemoteStoreNodeStats implements Writeable, ToXContentFragment { + + public static final String STATS_NAME = "remote_store"; + public static final String LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS = "last_successful_fetch_of_pinned_timestamps"; + + /** + * Time stamp for the last successful fetch of pinned timestamps by the {@linkplain RemoteStorePinnedTimestampService} + */ + private final long lastSuccessfulFetchOfPinnedTimestamps; + + public RemoteStoreNodeStats() { + this.lastSuccessfulFetchOfPinnedTimestamps = RemoteStorePinnedTimestampService.getPinnedTimestamps().v1(); + } + + public long getLastSuccessfulFetchOfPinnedTimestamps() { + return this.lastSuccessfulFetchOfPinnedTimestamps; + } + + public RemoteStoreNodeStats(StreamInput in) throws IOException { + this.lastSuccessfulFetchOfPinnedTimestamps = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(this.lastSuccessfulFetchOfPinnedTimestamps); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(STATS_NAME); + builder.field(LAST_SUCCESSFUL_FETCH_OF_PINNED_TIMESTAMPS, this.lastSuccessfulFetchOfPinnedTimestamps); + return builder.endObject(); + } + + @Override + public String toString() { + return "RemoteStoreNodeStats{ lastSuccessfulFetchOfPinnedTimestamps=" + lastSuccessfulFetchOfPinnedTimestamps + "}"; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (o.getClass() != RemoteStoreNodeStats.class) { + return false; + } + RemoteStoreNodeStats other = (RemoteStoreNodeStats) o; + return this.lastSuccessfulFetchOfPinnedTimestamps == other.lastSuccessfulFetchOfPinnedTimestamps; + } + + @Override + public int hashCode() { + return Objects.hash(lastSuccessfulFetchOfPinnedTimestamps); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 11902728eed07..34065daff2b8a 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -95,6 +95,7 @@ import org.opensearch.node.NodeResourceUsageStats; import org.opensearch.node.NodesResourceUsageStats; import org.opensearch.node.ResponseCollectorService; +import org.opensearch.node.remotestore.RemoteStoreNodeStats; import org.opensearch.ratelimitting.admissioncontrol.controllers.AdmissionController; import org.opensearch.ratelimitting.admissioncontrol.controllers.CpuBasedAdmissionController; import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType; @@ -614,6 +615,14 @@ public void testSerialization() throws IOException { } else { assertEquals(nodeCacheStats, deserializedNodeCacheStats); } + + RemoteStoreNodeStats remoteStoreNodeStats = nodeStats.getRemoteStoreNodeStats(); + RemoteStoreNodeStats deserializedRemoteStoreNodeStats = deserializedNodeStats.getRemoteStoreNodeStats(); + if (remoteStoreNodeStats == null) { + assertNull(deserializedRemoteStoreNodeStats); + } else { + assertEquals(remoteStoreNodeStats, deserializedRemoteStoreNodeStats); + } } } } @@ -996,6 +1005,16 @@ public void apply(String action, AdmissionControlActionType admissionControlActi nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags); } + RemoteStoreNodeStats remoteStoreNodeStats = null; + if (frequently()) { + remoteStoreNodeStats = new RemoteStoreNodeStats() { + @Override + public long getLastSuccessfulFetchOfPinnedTimestamps() { + return 123456L; + } + }; + } + // TODO: Only remote_store based aspects of NodeIndicesStats are being tested here. // It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now return new NodeStats( @@ -1027,7 +1046,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi segmentReplicationRejectionStats, null, admissionControlStats, - nodeCacheStats + nodeCacheStats, + remoteStoreNodeStats ); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index 1c4a77905d73f..823661ba14abf 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -349,6 +349,7 @@ private ClusterStatsNodeResponse createClusterStatsNodeResponse( null, null, null, + null, null ); if (defaultBehavior) { diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index 5539dd26dd52d..cd050fb346563 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -195,6 +195,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -226,6 +227,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -257,6 +259,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ) ); @@ -319,6 +322,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -350,6 +354,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -381,6 +386,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ) ); diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index 35ca5d80aeb4e..ded457601c0ae 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -125,7 +125,8 @@ List adjustNodesStats(List nodesStats) { nodeStats.getSegmentReplicationRejectionStats(), nodeStats.getRepositoriesStats(), nodeStats.getAdmissionControlStats(), - nodeStats.getNodeCacheStats() + nodeStats.getNodeCacheStats(), + nodeStats.getRemoteStoreNodeStats() ); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 7adff82e72245..fa5fb736f518f 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2752,6 +2752,7 @@ public void ensureEstimatedStats() { false, false, false, + false, false ); assertThat( From c46bf412fd046b9b3f7bbd6e8ba10e2fc57dcfa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:33:35 -0400 Subject: [PATCH 172/357] Bump org.jline:jline from 3.26.3 to 3.27.0 in /test/fixtures/hdfs-fixture (#16135) * Bump org.jline:jline in /test/fixtures/hdfs-fixture Bumps [org.jline:jline](https://github.com/jline/jline3) from 3.26.3 to 3.27.0. - [Release notes](https://github.com/jline/jline3/releases) - [Changelog](https://github.com/jline/jline3/blob/master/changelog.md) - [Commits](https://github.com/jline/jline3/compare/jline-parent-3.26.3...jline-3.27.0) --- updated-dependencies: - dependency-name: org.jline:jline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0859a7d5af8..3df8aec0f376a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) +- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 99b0386a48808..bc63362980bea 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -77,7 +77,7 @@ dependencies { api "ch.qos.logback:logback-core:1.5.8" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" - api 'org.jline:jline:3.26.3' + api 'org.jline:jline:3.27.0' api 'org.apache.commons:commons-configuration2:2.11.0' api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api ('org.apache.kerby:kerb-admin:2.1.0') { From 7b248efc5ab50573ed25a9eaac5d44c9b75c243a Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 1 Oct 2024 12:09:02 -0400 Subject: [PATCH 173/357] Remove Identity FeatureFlag (#16024) * Remove Identity FeatureFlag Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Mark IdentityService as internal Signed-off-by: Craig Perkins * Add @InternalApi annotation Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 3 ++- .../org/opensearch/http/IdentityAuthenticationIT.java | 1 - .../opensearch/common/settings/FeatureFlagSettings.java | 1 - .../java/org/opensearch/common/util/FeatureFlags.java | 8 -------- .../java/org/opensearch/identity/IdentityService.java | 4 +++- server/src/main/java/org/opensearch/node/Node.java | 6 +----- .../org/opensearch/common/util/FeatureFlagTests.java | 9 +++------ 7 files changed, 9 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df8aec0f376a..ef56029b31eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) -- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) @@ -37,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) +- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) ### Deprecated diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java index 1a806b033eb8a..14346b8910c76 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/IdentityAuthenticationIT.java @@ -38,7 +38,6 @@ public class IdentityAuthenticationIT extends HttpSmokeTestCase { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(FeatureFlags.IDENTITY, "true") .build(); } diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 73375f9eaa413..59d999798868e 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -31,7 +31,6 @@ protected FeatureFlagSettings( public static final Set> BUILT_IN_FEATURE_FLAGS = Set.of( FeatureFlags.EXTENSIONS_SETTING, - FeatureFlags.IDENTITY_SETTING, FeatureFlags.TELEMETRY_SETTING, FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.TIERED_REMOTE_INDEX_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index a5acea004c3b2..6df68013a8119 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -40,11 +40,6 @@ public class FeatureFlags { */ public static final String EXTENSIONS = "opensearch.experimental.feature.extensions.enabled"; - /** - * Gates the functionality of identity. - */ - public static final String IDENTITY = "opensearch.experimental.feature.identity.enabled"; - /** * Gates the functionality of telemetry framework. */ @@ -82,8 +77,6 @@ public class FeatureFlags { public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); - public static final Setting IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope); - public static final Setting TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope); public static final Setting DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting( @@ -138,7 +131,6 @@ public class FeatureFlags { private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, - IDENTITY_SETTING, TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, diff --git a/server/src/main/java/org/opensearch/identity/IdentityService.java b/server/src/main/java/org/opensearch/identity/IdentityService.java index 03f937180f4ba..33066fae5a80d 100644 --- a/server/src/main/java/org/opensearch/identity/IdentityService.java +++ b/server/src/main/java/org/opensearch/identity/IdentityService.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.settings.Settings; import org.opensearch.identity.noop.NoopIdentityPlugin; import org.opensearch.identity.tokens.TokenManager; @@ -22,8 +23,9 @@ /** * Identity and access control for OpenSearch * - * @opensearch.experimental + * @opensearch.internal * */ +@InternalApi public class IdentityService { private static final Logger log = LogManager.getLogger(IdentityService.class); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 4962d72d8728a..56d04df5921ee 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -526,11 +526,7 @@ protected Node( FeatureFlags.initializeFeatureFlags(settings); final List identityPlugins = new ArrayList<>(); - if (FeatureFlags.isEnabled(FeatureFlags.IDENTITY)) { - // If identity is enabled load plugins implementing the extension point - logger.info("Identity on so found plugins implementing: " + pluginsService.filterPlugins(IdentityPlugin.class).toString()); - identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); - } + identityPlugins.addAll(pluginsService.filterPlugins(IdentityPlugin.class)); final Set additionalRoles = pluginsService.filterPlugins(Plugin.class) .stream() diff --git a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java index 6b6c7b96266d3..6d9d1aad3c5d5 100644 --- a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java +++ b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java @@ -14,7 +14,6 @@ import static org.opensearch.common.util.FeatureFlags.DATETIME_FORMATTER_CACHING; import static org.opensearch.common.util.FeatureFlags.EXTENSIONS; -import static org.opensearch.common.util.FeatureFlags.IDENTITY; public class FeatureFlagTests extends OpenSearchTestCase { @@ -40,7 +39,7 @@ public void testNonBooleanFeatureFlag() { } public void testBooleanFeatureFlagWithDefaultSetToFalse() { - final String testFlag = IDENTITY; + final String testFlag = EXTENSIONS; FeatureFlags.initializeFeatureFlags(Settings.EMPTY); assertNotNull(testFlag); assertFalse(FeatureFlags.isEnabled(testFlag)); @@ -49,15 +48,13 @@ public void testBooleanFeatureFlagWithDefaultSetToFalse() { public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToFalse() { final String testFlag = DATETIME_FORMATTER_CACHING; FeatureFlags.initializeFeatureFlags(Settings.EMPTY); - assertNotNull(testFlag); assertFalse(FeatureFlags.isEnabled(testFlag)); } public void testInitializeFeatureFlagsWithExperimentalSettings() { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(IDENTITY, true).build()); - assertTrue(FeatureFlags.isEnabled(IDENTITY)); + FeatureFlags.initializeFeatureFlags(Settings.builder().put(EXTENSIONS, true).build()); + assertTrue(FeatureFlags.isEnabled(EXTENSIONS)); assertFalse(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING)); - assertFalse(FeatureFlags.isEnabled(EXTENSIONS)); // reset FeatureFlags to defaults FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } From 908fefeed32ce205e7b6128a5a344bbf333a5e62 Mon Sep 17 00:00:00 2001 From: Arpit-Bandejiya Date: Tue, 1 Oct 2024 22:01:59 +0530 Subject: [PATCH 174/357] Change default retry mechanism of s3 client to Standard Mode (#15978) Signed-off-by: Arpit Bandejiya --- CHANGELOG.md | 1 + .../java/org/opensearch/repositories/s3/S3AsyncService.java | 5 ++++- .../main/java/org/opensearch/repositories/s3/S3Service.java | 3 +++ .../opensearch/repositories/s3/AwsS3ServiceImplTests.java | 3 ++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef56029b31eae..14232ad87478e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) +- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java index d691cad9c9d03..8bbef168de89c 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java @@ -18,6 +18,7 @@ import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; @@ -239,7 +240,9 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett RetryPolicy.builder() .numRetries(clientSettings.maxRetries) .throttlingBackoffStrategy( - clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy() : BackoffStrategy.none() + clientSettings.throttleRetries + ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD) + : BackoffStrategy.none() ) .build() ) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index fe81da31432f4..3d5e121778ba9 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.SystemPropertyTlsKeyManagersProvider; @@ -330,6 +331,8 @@ static ClientOverrideConfiguration buildOverrideConfiguration(final S3ClientSett ); if (!clientSettings.throttleRetries) { retryPolicy.throttlingBackoffStrategy(BackoffStrategy.none()); + } else { + retryPolicy.throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)); } return clientOverrideConfiguration.retryPolicy(retryPolicy.build()).build(); } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java index b80b857644f2a..e7312157d7a33 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryMode; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.apache.ProxyConfiguration; @@ -364,7 +365,7 @@ private void launchAWSConfigurationTest( if (expectedUseThrottleRetries) { assertThat( clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(), - is(BackoffStrategy.defaultThrottlingStrategy()) + is(BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD)) ); } else { assertThat(clientOverrideConfiguration.retryPolicy().get().throttlingBackoffStrategy(), is(BackoffStrategy.none())); From 0b1650de442d7c84f532bd23ae8e766abf1d5c45 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 1 Oct 2024 16:40:38 -0400 Subject: [PATCH 175/357] Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler (#16154) * Ensure RestHandler.Wrapper delegates all implementations to the wrapper handler Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Fix typo Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../java/org/opensearch/rest/RestHandler.java | 10 +++++ .../opensearch/rest/BaseRestHandlerTests.java | 41 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14232ad87478e..2fb5fd6f2757e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) +- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 143cbd472ed07..7c3a369be61b9 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -192,6 +192,16 @@ public List replacedRoutes() { public boolean allowSystemIndexAccessByDefault() { return delegate.allowSystemIndexAccessByDefault(); } + + @Override + public boolean isActionPaginated() { + return delegate.isActionPaginated(); + } + + @Override + public boolean supportsStreaming() { + return delegate.supportsStreaming(); + } } /** diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java index 45653e9d8e4d6..7534dcd93944a 100644 --- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java +++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java @@ -35,6 +35,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.RestHandler.ReplacedRoute; import org.opensearch.rest.RestHandler.Route; import org.opensearch.rest.RestRequest.Method; @@ -46,15 +48,22 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.lang.reflect.Modifier; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static org.hamcrest.core.StringContains.containsString; import static org.hamcrest.object.HasToString.hasToString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; public class BaseRestHandlerTests extends OpenSearchTestCase { private NodeClient mockClient; @@ -288,4 +297,36 @@ public void testReplaceRoutesMethod() throws Exception { } } + public void testRestHandlerWrapper() throws Exception { + RestHandler rh = new RestHandler() { + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); + } + }; + RestHandler handlerSpy = spy(rh); + RestHandler.Wrapper rhWrapper = new RestHandler.Wrapper(handlerSpy); + + List overridableMethods = Arrays.stream(RestHandler.class.getMethods()) + .filter( + m -> !(Modifier.isPrivate(m.getModifiers()) || Modifier.isStatic(m.getModifiers()) || Modifier.isFinal(m.getModifiers())) + ) + .collect(Collectors.toList()); + + for (java.lang.reflect.Method method : overridableMethods) { + int argCount = method.getParameterCount(); + Object[] args = new Object[argCount]; + for (int i = 0; i < argCount; i++) { + args[i] = any(); + } + if (args.length > 0) { + method.invoke(rhWrapper, args); + } else { + method.invoke(rhWrapper); + } + method.invoke(verify(handlerSpy, times(1)), args); + } + verifyNoMoreInteractions(handlerSpy); + } + } From 1ee858fc8cc631d35c4a48b2a53d4ab6009e2c2b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 1 Oct 2024 19:33:06 -0400 Subject: [PATCH 176/357] [BUG] Streaming bulk request hangs (#16158) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../rest/ReactorNetty4StreamingIT.java | 28 +++++++++++++++++++ .../org/opensearch/rest/RestController.java | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb5fd6f2757e..333b1cd19b404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) +- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) ### Security diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java index 6f3895fffa437..1b60023da0329 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Locale; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -297,4 +298,31 @@ public void testStreamingBadStream() throws IOException { assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200)); assertThat(streamingResponse.getWarnings(), empty()); } + + public void testStreamingLargeDocument() throws IOException { + final Stream stream = Stream.of( + String.format( + Locale.getDefault(), + "{ \"index\": { \"_index\": \"test-streaming\", \"_id\": \"1\" } }\n{ \"name\": \"%s\" }\n", + randomAlphaOfLength(5000) + ) + ); + + final Duration delay = Duration.ofMillis(1); + final StreamingRequest streamingRequest = new StreamingRequest<>( + "POST", + "/_bulk/stream", + Flux.fromStream(stream).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + ); + + final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); + + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"type\":\"illegal_argument_exception\"")) + .expectComplete() + .verify(); + + assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(streamingResponse.getWarnings(), empty()); + } } diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 4f87c01258396..c17f723c13f2a 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -709,7 +709,7 @@ public void sendResponse(RestResponse response) { prepareResponse(response.status(), Map.of("Content-Type", List.of(response.contentType()))); } - Mono.ignoreElements(this).then(Mono.just(response)).subscribe(delegate::sendResponse); + Mono.from(this).ignoreElement().then(Mono.just(response)).subscribe(delegate::sendResponse); } @Override From 79a2ec1daa06296ebfe6ca23e6fc64f720d2d9f4 Mon Sep 17 00:00:00 2001 From: prachi-gaonkar Date: Wed, 2 Oct 2024 18:51:51 +0530 Subject: [PATCH 177/357] Fix client-rest-high-level tests for ppc64le (#16089) * Fix client-rest-high-level tests for ppc64le Signed-off-by: prachi-gaonkar * Fix client-rest-high-level tests for ppc64le Signed-off-by: prachi-gaonkar --------- Signed-off-by: prachi-gaonkar --- .../client/OpenSearchRestHighLevelClientTestCase.java | 1 + .../src/test/java/org/opensearch/client/RankEvalIT.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java index b0a7d1e3578c0..b512117c42f65 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java @@ -90,6 +90,7 @@ public abstract class OpenSearchRestHighLevelClientTestCase extends OpenSearchRestTestCase { protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline"; + protected static final double DOUBLE_DELTA = 0.000001; private static RestHighLevelClient restHighLevelClient; private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false")); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java index 01fdd489aa7d8..6da8a29a9789e 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java @@ -158,7 +158,7 @@ public void testMetrics() throws IOException { RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" }); RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); - assertEquals(expectedScores[i], response.getMetricScore(), Double.MIN_VALUE); + assertEquals(expectedScores[i], response.getMetricScore(), DOUBLE_DELTA); i++; } } From 4e3a6d038dfc37660c7b9e47f570a32b0f2be4d4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:56:13 -0400 Subject: [PATCH 178/357] [AUTO] [main] Add bwc version 2.17.2. (#16166) * Add bwc version 2.17.2 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update Version.java Signed-off-by: Andriy Redko --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Andriy Redko Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com> Co-authored-by: Andriy Redko --- .ci/bwcVersions | 1 + README.md | 2 +- libs/core/src/main/java/org/opensearch/Version.java | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index ccb47c1a3b724..5587b8e5784c5 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -39,4 +39,5 @@ BWC_VERSION: - "2.16.1" - "2.17.0" - "2.17.1" + - "2.17.2" - "2.18.0" diff --git a/README.md b/README.md index 03728aee0135c..95fbac7bbecf1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) [![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") -[![2.17.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.1") +[![2.17.2 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.2)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.2") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index b86b8459fb8a8..5d38f85c40c89 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -110,6 +110,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From e7ad37db8037409a2ca4e6660a7faa0651a60c7e Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:01:14 +0530 Subject: [PATCH 179/357] Change version checks from CURRENT to 2.18 (#16174) Signed-off-by: Lakshya Taragi --- .../action/admin/cluster/node/stats/NodeStats.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index c91260778f037..eb79e3403a25c 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -247,8 +247,7 @@ public NodeStats(StreamInput in) throws IOException { } else { nodeCacheStats = null; } - // TODO: change version to V_2_18_0 - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { remoteStoreNodeStats = in.readOptionalWriteable(RemoteStoreNodeStats::new); } else { remoteStoreNodeStats = null; @@ -542,8 +541,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeOptionalWriteable(nodeCacheStats); } - // TODO: change version to V_2_18_0 - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalWriteable(remoteStoreNodeStats); } } From bf6566e76c46bd7dc5d838503b8c43ff8cb09757 Mon Sep 17 00:00:00 2001 From: Sumit Bansal Date: Thu, 3 Oct 2024 13:29:26 +0530 Subject: [PATCH 180/357] Add changes to block calls in cat shards, indices and segments based on dynamic limit settings (#15986) * Add changes to block calls in cat shards, indices and segments based on dynamic limit settings Signed-off-by: Sumit Bansal --- CHANGELOG.md | 1 + .../opensearch/OpenSearchServerException.java | 9 + .../org/opensearch/action/ActionModule.java | 11 +- .../cluster/shards/CatShardsRequest.java | 10 + .../shards/TransportCatShardsAction.java | 31 ++- .../ResponseLimitBreachedException.java | 66 +++++ .../common/breaker/ResponseLimitSettings.java | 180 ++++++++++++++ .../common/settings/ClusterSettings.java | 8 +- .../rest/action/cat/AbstractCatAction.java | 9 + .../rest/action/cat/RestIndicesAction.java | 25 +- .../rest/action/cat/RestSegmentsAction.java | 25 ++ .../rest/action/cat/RestShardsAction.java | 6 + .../action/list/RestIndicesListAction.java | 10 + .../ExceptionSerializationTests.java | 2 + .../RenamedTimeoutRequestParameterTests.java | 12 +- .../breaker/ResponseLimitSettingsTests.java | 228 ++++++++++++++++++ .../action/cat/RestIndicesActionTests.java | 14 +- 17 files changed, 636 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java create mode 100644 server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java create mode 100644 server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 333b1cd19b404..b69c33df3b6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) +- Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index b0ab62259d5c0..0ee3debb4819a 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -14,6 +14,7 @@ import static org.opensearch.Version.V_2_10_0; import static org.opensearch.Version.V_2_13_0; import static org.opensearch.Version.V_2_17_0; +import static org.opensearch.Version.V_2_18_0; import static org.opensearch.Version.V_2_1_0; import static org.opensearch.Version.V_2_4_0; import static org.opensearch.Version.V_2_5_0; @@ -1210,6 +1211,14 @@ public static void registerExceptions() { V_2_17_0 ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.common.breaker.ResponseLimitBreachedException.class, + org.opensearch.common.breaker.ResponseLimitBreachedException::new, + 175, + V_2_18_0 + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 937d7509fe68c..98bcd6ba9c3f8 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -300,6 +300,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.NamedRegistry; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.inject.AbstractModule; import org.opensearch.common.inject.TypeLiteral; import org.opensearch.common.inject.multibindings.MapBinder; @@ -531,6 +532,7 @@ public class ActionModule extends AbstractModule { private final RequestValidators indicesAliasesRequestRequestValidators; private final ThreadPool threadPool; private final ExtensionsManager extensionsManager; + private final ResponseLimitSettings responseLimitSettings; public ActionModule( Settings settings, @@ -583,6 +585,7 @@ public ActionModule( ); restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService); + responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); } public Map> getActions() { @@ -968,8 +971,8 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClusterManagerAction()); registerHandler.accept(new RestNodesAction()); registerHandler.accept(new RestTasksAction(nodesInCluster)); - registerHandler.accept(new RestIndicesAction()); - registerHandler.accept(new RestSegmentsAction()); + registerHandler.accept(new RestIndicesAction(responseLimitSettings)); + registerHandler.accept(new RestSegmentsAction(responseLimitSettings)); // Fully qualified to prevent interference with rest.action.count.RestCountAction registerHandler.accept(new org.opensearch.rest.action.cat.RestCountAction()); // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction @@ -989,7 +992,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestTemplatesAction()); // LIST API - registerHandler.accept(new RestIndicesListAction()); + registerHandler.accept(new RestIndicesListAction(responseLimitSettings)); // Point in time API registerHandler.accept(new RestCreatePitAction()); @@ -1060,6 +1063,8 @@ protected void configure() { // register dynamic ActionType -> transportAction Map used by NodeClient bind(DynamicActionRegistry.class).toInstance(dynamicActionRegistry); + + bind(ResponseLimitSettings.class).toInstance(responseLimitSettings); } public ActionFilters getActionFilters() { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java index 49299777db8ae..76aa25b9a96b5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java @@ -27,11 +27,13 @@ public class CatShardsRequest extends ClusterManagerNodeReadRequest headers) { return new ClusterAdminTask(id, type, action, parentTaskId, headers, this.cancelAfterTimeInterval); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java index 224d3cbc5f10a..6b073a16f7d28 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java @@ -16,6 +16,8 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TimeoutTaskCancellationUtility; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.NotifyOnceListener; @@ -23,6 +25,10 @@ import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +import java.util.Objects; + +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.SHARDS; + /** * Perform cat shards action * @@ -31,11 +37,18 @@ public class TransportCatShardsAction extends HandledTransportAction { private final NodeClient client; + private final ResponseLimitSettings responseLimitSettings; @Inject - public TransportCatShardsAction(NodeClient client, TransportService transportService, ActionFilters actionFilters) { + public TransportCatShardsAction( + NodeClient client, + TransportService transportService, + ActionFilters actionFilters, + ResponseLimitSettings responseLimitSettings + ) { super(CatShardsAction.NAME, transportService, actionFilters, CatShardsRequest::new); this.client = client; + this.responseLimitSettings = responseLimitSettings; } @Override @@ -73,6 +86,7 @@ protected void innerOnFailure(Exception e) { client.admin().cluster().state(clusterStateRequest, new ActionListener() { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { + validateRequestLimit(shardsRequest, clusterStateResponse, cancellableListener); catShardsResponse.setClusterStateResponse(clusterStateResponse); IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); indicesStatsRequest.setShouldCancelOnTimeout(true); @@ -107,4 +121,19 @@ public void onFailure(Exception e) { } } + + private void validateRequestLimit( + final CatShardsRequest shardsRequest, + final ClusterStateResponse clusterStateResponse, + final ActionListener listener + ) { + if (shardsRequest.isRequestLimitCheckSupported() + && Objects.nonNull(clusterStateResponse) + && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatShardsResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), SHARDS, limit)) { + listener.onFailure(new ResponseLimitBreachedException("Too many shards requested.", limit, SHARDS)); + } + } + } } diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java new file mode 100644 index 0000000000000..db6785067edf6 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitBreachedException.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.OpenSearchException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Thrown when api response breaches threshold limit. + * + * @opensearch.internal + */ +public class ResponseLimitBreachedException extends OpenSearchException { + + private final int responseLimit; + private final ResponseLimitSettings.LimitEntity limitEntity; + + public ResponseLimitBreachedException(StreamInput in) throws IOException { + super(in); + responseLimit = in.readVInt(); + limitEntity = in.readEnum(ResponseLimitSettings.LimitEntity.class); + } + + public ResponseLimitBreachedException(String msg, int responseLimit, ResponseLimitSettings.LimitEntity limitEntity) { + super(msg); + this.responseLimit = responseLimit; + this.limitEntity = limitEntity; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(responseLimit); + out.writeEnum(limitEntity); + } + + public int getResponseLimit() { + return responseLimit; + } + + public ResponseLimitSettings.LimitEntity getLimitEntity() { + return limitEntity; + } + + @Override + public RestStatus status() { + return RestStatus.TOO_MANY_REQUESTS; + } + + @Override + protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("response_limit", responseLimit); + builder.field("limit_entity", limitEntity); + } +} diff --git a/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java new file mode 100644 index 0000000000000..8eb47c9aaf147 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/breaker/ResponseLimitSettings.java @@ -0,0 +1,180 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.rest.action.cat.RestIndicesAction; +import org.opensearch.rest.action.cat.RestSegmentsAction; +import org.opensearch.rest.action.cat.RestShardsAction; + +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +/** + * Class to define dynamic settings for putting response limits on the actions and methods to evaluate if block is required. + */ +public class ResponseLimitSettings { + + /** + * Enum to represent entity against which we need to perform limit checks. + */ + public enum LimitEntity { + INDICES, + SHARDS + } + + private volatile int catIndicesResponseLimit; + private volatile int catShardsResponseLimit; + private volatile int catSegmentsResponseLimit; + + /** + * Setting to enable response limit on {@link RestIndicesAction}. The limit will be applied on number of indices. + */ + public static final Setting CAT_INDICES_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.indices.response.limit.number_of_indices", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Setting to enable response limit on {@link RestShardsAction}. The limit will be applied on number of shards. + */ + public static final Setting CAT_SHARDS_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.shards.response.limit.number_of_shards", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + /** + * Setting to enable response limit on {@link RestSegmentsAction}. The limit will be applied on number of indices. + */ + public static final Setting CAT_SEGMENTS_RESPONSE_LIMIT_SETTING = Setting.intSetting( + "cat.segments.response.limit.number_of_indices", + -1, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public ResponseLimitSettings(ClusterSettings clusterSettings, Settings settings) { + setCatShardsResponseLimit(CAT_SHARDS_RESPONSE_LIMIT_SETTING.get(settings)); + setCatIndicesResponseLimit(CAT_INDICES_RESPONSE_LIMIT_SETTING.get(settings)); + setCatSegmentsResponseLimit(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.get(settings)); + + clusterSettings.addSettingsUpdateConsumer(CAT_SHARDS_RESPONSE_LIMIT_SETTING, this::setCatShardsResponseLimit); + clusterSettings.addSettingsUpdateConsumer(CAT_INDICES_RESPONSE_LIMIT_SETTING, this::setCatIndicesResponseLimit); + clusterSettings.addSettingsUpdateConsumer(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING, this::setCatSegmentsResponseLimit); + } + + /** + * Method to check if the response limit has reached for an action. + * The limits are controlled via dynamic settings. + * + * @param metadata {@link Metadata} + * @param limitEntity {@link LimitEntity} + * @param limit Integer limit on block entity + * @return True/False + */ + public static boolean isResponseLimitBreached(final Metadata metadata, final LimitEntity limitEntity, final int limit) { + if (Objects.isNull(metadata) || limit <= 0) return false; + if (limitEntity == LimitEntity.INDICES) { + int indicesCount = getTotalIndicesFromMetadata.apply(metadata); + return indicesCount > limit; + } else { + throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]"); + } + } + + /** + * Method to check if the response limit has reached for an action. + * The limits are controlled via dynamic settings. + * + * @param routingTable {@link RoutingTable} + * @param limitEntity {@link LimitEntity} + * @param limit Integer limit on block entity + * @return True/False + */ + public static boolean isResponseLimitBreached(final RoutingTable routingTable, final LimitEntity limitEntity, final int limit) { + if (Objects.isNull(routingTable) || limit <= 0) return false; + if (Objects.isNull(limitEntity)) { + throw new IllegalArgumentException("Limit entity cannot be null"); + } + switch (limitEntity) { + case INDICES: + int indicesCount = getTotalIndicesFromRoutingTable.apply(routingTable); + if (indicesCount > limit) return true; + break; + case SHARDS: + if (isShardsLimitBreached(routingTable, limit)) return true; + break; + default: + throw new IllegalArgumentException("Unsupported limit entity [" + limitEntity + "]"); + } + return false; + } + + private static boolean isShardsLimitBreached(final RoutingTable routingTable, final int limit) { + final Map indexRoutingTableMap = routingTable.getIndicesRouting(); + int totalShards = 0; + for (final Map.Entry entry : indexRoutingTableMap.entrySet()) { + for (final Map.Entry indexShardRoutingTableEntry : entry.getValue().getShards().entrySet()) { + totalShards += indexShardRoutingTableEntry.getValue().getShards().size(); + // Fail fast if limit value is breached and avoid unnecessary computation. + if (totalShards > limit) return true; + } + } + return false; + } + + private void setCatShardsResponseLimit(final int catShardsResponseLimit) { + this.catShardsResponseLimit = catShardsResponseLimit; + } + + private void setCatIndicesResponseLimit(final int catIndicesResponseLimit) { + this.catIndicesResponseLimit = catIndicesResponseLimit; + } + + private void setCatSegmentsResponseLimit(final int catSegmentsResponseLimit) { + this.catSegmentsResponseLimit = catSegmentsResponseLimit; + } + + public int getCatShardsResponseLimit() { + return this.catShardsResponseLimit; + } + + public int getCatIndicesResponseLimit() { + return this.catIndicesResponseLimit; + } + + public int getCatSegmentsResponseLimit() { + return this.catSegmentsResponseLimit; + } + + static Function getTotalIndicesFromMetadata = (metadata) -> { + if (Objects.nonNull(metadata) && Objects.nonNull(metadata.getIndices())) { + return metadata.getIndices().size(); + } + return 0; + }; + + static Function getTotalIndicesFromRoutingTable = (routingTable) -> { + if (Objects.nonNull(routingTable) && Objects.nonNull(routingTable.getIndicesRouting())) { + return routingTable.getIndicesRouting().size(); + } + return 0; + }; +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index ecdd23530c648..e18c4dabb29eb 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -84,6 +84,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.settings.CacheSettings; import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings; @@ -796,7 +797,12 @@ public void apply(Settings value, Settings current, Settings previous) { WorkloadManagementSettings.NODE_LEVEL_CPU_REJECTION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_CPU_CANCELLATION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_MEMORY_REJECTION_THRESHOLD, - WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD + WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD, + + // Settings to be used for limiting rest requests + ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING, + ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING, + ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index 6f4e060363bfb..506f5f1529776 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -33,6 +33,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.io.Streams; import org.opensearch.common.io.UTF8StreamWriter; import org.opensearch.core.common.io.stream.BytesStream; @@ -98,4 +99,12 @@ protected Set responseParams() { return RESPONSE_PARAMS; } + /** + * Method to check if limits defined in {@link ResponseLimitSettings} are applicable to an action. + * + * @return True / False status + */ + protected boolean isRequestLimitCheckSupported() { + return false; + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 1e76008ff8c64..7562e09ddd704 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -50,6 +50,8 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; @@ -86,6 +88,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest.DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT; +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES; import static org.opensearch.rest.RestRequest.Method.GET; /** @@ -102,6 +105,12 @@ public class RestIndicesAction extends AbstractListAction { private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private final ResponseLimitSettings responseLimitSettings; + + public RestIndicesAction(ResponseLimitSettings responseLimitSettings) { + this.responseLimitSettings = responseLimitSettings; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); @@ -123,6 +132,11 @@ protected void documentation(StringBuilder sb) { sb.append("/_cat/indices/{index}\n"); } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -162,7 +176,6 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the get settings request might be deleted when the // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases: // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we @@ -181,6 +194,7 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { new ActionListener() { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { + validateRequestLimit(clusterStateResponse, listener); IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from // rest request and unresolved, while for paginated queries, it would be a list of indices @@ -234,6 +248,15 @@ public void onFailure(final Exception e) { } + private void validateRequestLimit(final ClusterStateResponse clusterStateResponse, final ActionListener

listener) { + if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatIndicesResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getMetadata(), INDICES, limit)) { + listener.onFailure(new ResponseLimitBreachedException("Too many indices requested.", limit, INDICES)); + } + } + } + /** * We're using the Get Settings API here to resolve the authorized indices for the user. * This is because the Cluster State and Cluster Health APIs do not filter output based diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java index b88af4ac3eeed..69216cb46492d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java @@ -42,6 +42,8 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.Table; +import org.opensearch.common.breaker.ResponseLimitBreachedException; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.common.Strings; import org.opensearch.index.engine.Segment; @@ -52,9 +54,11 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; +import static org.opensearch.common.breaker.ResponseLimitSettings.LimitEntity.INDICES; import static org.opensearch.rest.RestRequest.Method.GET; /** @@ -66,6 +70,12 @@ public class RestSegmentsAction extends AbstractCatAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestSegmentsAction.class); + private final ResponseLimitSettings responseLimitSettings; + + public RestSegmentsAction(ResponseLimitSettings responseLimitSettings) { + this.responseLimitSettings = responseLimitSettings; + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_cat/segments"), new Route(GET, "/_cat/segments/{index}"))); @@ -81,6 +91,11 @@ public boolean allowSystemIndexAccessByDefault() { return true; } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -96,6 +111,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener(channel) { @Override public void processResponse(final ClusterStateResponse clusterStateResponse) { + validateRequestLimit(clusterStateResponse); final IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest(); indicesSegmentsRequest.indices(indices); client.admin().indices().segments(indicesSegmentsRequest, new RestResponseListener(channel) { @@ -110,6 +126,15 @@ public RestResponse buildResponse(final IndicesSegmentResponse indicesSegmentRes }); } + private void validateRequestLimit(final ClusterStateResponse clusterStateResponse) { + if (isRequestLimitCheckSupported() && Objects.nonNull(clusterStateResponse) && Objects.nonNull(clusterStateResponse.getState())) { + int limit = responseLimitSettings.getCatSegmentsResponseLimit(); + if (ResponseLimitSettings.isResponseLimitBreached(clusterStateResponse.getState().getRoutingTable(), INDICES, limit)) { + throw new ResponseLimitBreachedException("Segments from too many indices requested.", limit, INDICES); + } + } + } + @Override protected void documentation(StringBuilder sb) { sb.append("/_cat/segments\n"); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index a7ad5fe6c14a3..c8b4e7472927e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -105,6 +105,11 @@ protected void documentation(StringBuilder sb) { sb.append("/_cat/shards/{index}\n"); } + @Override + public boolean isRequestLimitCheckSupported() { + return true; + } + @Override public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); @@ -113,6 +118,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli shardsRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", shardsRequest.clusterManagerNodeTimeout())); shardsRequest.setCancelAfterTimeInterval(request.paramAsTime("cancel_after_time_interval", NO_TIMEOUT)); shardsRequest.setIndices(indices); + shardsRequest.setRequestLimitCheckSupported(isRequestLimitCheckSupported()); parseDeprecatedMasterTimeoutParameter(shardsRequest, request, deprecationLogger, getName()); return channel -> client.execute(CatShardsAction.INSTANCE, shardsRequest, new RestResponseListener(channel) { @Override diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index ad5c58c86ce90..971cab0535be9 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,6 +9,7 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; @@ -35,6 +36,10 @@ public class RestIndicesListAction extends RestIndicesAction { private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; + public RestIndicesListAction(final ResponseLimitSettings responseLimitSettings) { + super(responseLimitSettings); + } + @Override public List routes() { return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}"))); @@ -70,6 +75,11 @@ protected PageParams validateAndGetPageParams(RestRequest restRequest) { return pageParams; } + @Override + public boolean isRequestLimitCheckSupported() { + return false; + } + protected int defaultPageSize() { return DEFAULT_LIST_INDICES_PAGE_SIZE; } diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2e4a2d7bdd59c..c1972daeab6d3 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -65,6 +65,7 @@ import org.opensearch.cluster.routing.UnsupportedWeightedRoutingStateException; import org.opensearch.cluster.service.ClusterManagerThrottlingException; import org.opensearch.common.UUIDs; +import org.opensearch.common.breaker.ResponseLimitBreachedException; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -898,6 +899,7 @@ public void testIds() { ids.put(172, ViewNotFoundException.class); ids.put(173, ViewAlreadyExistsException.class); ids.put(174, InvalidIndexContextException.class); + ids.put(175, ResponseLimitBreachedException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java index c7e1039686cc9..26c4670b4288c 100644 --- a/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java +++ b/server/src/test/java/org/opensearch/action/RenamedTimeoutRequestParameterTests.java @@ -11,7 +11,9 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.core.common.bytes.BytesArray; @@ -155,7 +157,10 @@ public void testCatAllocation() { } public void testCatIndices() { - RestIndicesAction action = new RestIndicesAction(); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + Settings settings = Settings.builder().build(); + ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + RestIndicesAction action = new RestIndicesAction(responseLimitSettings); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); @@ -239,7 +244,10 @@ public void testCatThreadPool() { } public void testCatSegments() { - RestSegmentsAction action = new RestSegmentsAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + RestSegmentsAction action = new RestSegmentsAction(responseLimitSettings); Exception e = assertThrows(OpenSearchParseException.class, () -> action.doCatRequest(getRestRequestWithBothParams(), client)); assertThat(e.getMessage(), containsString(DUPLICATE_PARAMETER_ERROR_MESSAGE)); assertWarnings(MASTER_TIMEOUT_DEPRECATED_MESSAGE); diff --git a/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java new file mode 100644 index 0000000000000..4633d055b3505 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/breaker/ResponseLimitSettingsTests.java @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.breaker; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING; +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING; +import static org.opensearch.common.breaker.ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING; + +public class ResponseLimitSettingsTests extends OpenSearchTestCase { + + public void testIsResponseLimitBreachedForNullMetadataExpectNotBreached() { + final Settings settings = Settings.builder().build(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + (Metadata) null, + ResponseLimitSettings.LimitEntity.INDICES, + 0 + ); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForNullRoutingTableExpectNotBreached() { + final Settings settings = Settings.builder().build(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + (RoutingTable) null, + ResponseLimitSettings.LimitEntity.INDICES, + 0 + ); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForNullLimitEntityWithRoutingTableExpectException() { + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + expectThrows( + IllegalArgumentException.class, + () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getRoutingTable(), null, 4) + ); + } + + public void testIsResponseLimitBreachedForNullLimitEntityWithMetadataExpectException() { + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + expectThrows( + IllegalArgumentException.class, + () -> ResponseLimitSettings.isResponseLimitBreached(clusterState.getMetadata(), null, 4) + ); + } + + public void testIsResponseLimitBreachedForCatIndicesWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectBreached() { + // Set limit of 1 index + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 1).build(); + final boolean breached = isBreachedForIndices(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatIndicesWithSettingEnabledExpectNotBreached() { + // Set limit of 5 indices + final Settings settings = Settings.builder().put(CAT_INDICES_RESPONSE_LIMIT_SETTING.getKey(), 5).build(); + final boolean breached = isBreachedForIndices(settings); + assertFalse(breached); + } + + private static boolean isBreachedForIndices(final Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Pass cluster state with 3 indices + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + return ResponseLimitSettings.isResponseLimitBreached( + clusterState.getMetadata(), + ResponseLimitSettings.LimitEntity.INDICES, + responseLimitSettings.getCatIndicesResponseLimit() + ); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatShardsWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + private static boolean isBreachedForShards(Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Build cluster state with 3 shards + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + return ResponseLimitSettings.isResponseLimitBreached( + clusterState.getRoutingTable(), + ResponseLimitSettings.LimitEntity.SHARDS, + responseLimitSettings.getCatShardsResponseLimit() + ); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectBreached() { + // Set limit of 2 shards + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 2).build(); + final boolean breached = isBreachedForShards(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatShardsWithSettingEnabledExpectNotBreached() { + // Set limit of 9 shards + final Settings settings = Settings.builder().put(CAT_SHARDS_RESPONSE_LIMIT_SETTING.getKey(), 9).build(); + final boolean breached = isBreachedForShards(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingDisabledExpectNotBreached() { + // Don't enable limit + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), -1).build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithNoSettingPassedExpectNotBreached() { + // Don't set setting + final Settings settings = Settings.builder().build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + private static boolean isBreachedForSegments(Settings settings) { + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + // Build cluster state with 3 indices + final ClusterState clusterState = buildClusterState("test-index-1", "test-index-2", "test-index-3"); + final boolean breached = ResponseLimitSettings.isResponseLimitBreached( + clusterState.getRoutingTable(), + ResponseLimitSettings.LimitEntity.INDICES, + responseLimitSettings.getCatSegmentsResponseLimit() + ); + return breached; + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectBreached() { + // Set limit of 1 index + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 1).build(); + final boolean breached = isBreachedForSegments(settings); + assertTrue(breached); + } + + public void testIsResponseLimitBreachedForCatSegmentsWithSettingEnabledExpectNotBreached() { + // Set limit of 5 indices + final Settings settings = Settings.builder().put(CAT_SEGMENTS_RESPONSE_LIMIT_SETTING.getKey(), 5).build(); + final boolean breached = isBreachedForSegments(settings); + assertFalse(breached); + } + + private static ClusterState buildClusterState(String... indices) { + final Metadata.Builder metadata = Metadata.builder(); + for (String index : indices) { + metadata.put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)); + } + final Map indexRoutingTableMap = new HashMap<>(); + for (String s : indices) { + final Index index = new Index(s, "uuid"); + final ShardId primaryShardId = new ShardId(index, 0); + final ShardRouting primaryShardRouting = createShardRouting(primaryShardId, true); + final ShardId replicaShardId = new ShardId(index, 1); + final ShardRouting replicaShardRouting = createShardRouting(replicaShardId, false); + final IndexShardRoutingTable.Builder indexShardRoutingTableBuilder = new IndexShardRoutingTable.Builder(primaryShardId); + indexShardRoutingTableBuilder.addShard(primaryShardRouting); + indexShardRoutingTableBuilder.addShard(replicaShardRouting); + final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index) + .addShard(primaryShardRouting) + .addShard(replicaShardRouting) + .addIndexShard(indexShardRoutingTableBuilder.build()); + indexRoutingTableMap.put(index.getName(), indexRoutingTable.build()); + } + final RoutingTable routingTable = new RoutingTable(1, indexRoutingTableMap); + return ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(routingTable) + .build(); + } + + private static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { + return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); + } +} diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 1d1b509ae94e5..2debb00a0d14f 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -43,6 +43,8 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.Table; import org.opensearch.common.UUIDs; +import org.opensearch.common.breaker.ResponseLimitSettings; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; @@ -144,7 +146,10 @@ public void setup() { } public void testBuildTable() { - final RestIndicesAction action = new RestIndicesAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); final Table table = action.buildTable( new FakeRestRequest(), indicesSettings, @@ -165,8 +170,11 @@ public void testBuildTable() { } public void testBuildPaginatedTable() { - final RestIndicesAction action = new RestIndicesAction(); - final RestIndicesListAction indicesListAction = new RestIndicesListAction(); + final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + final Settings settings = Settings.builder().build(); + final ResponseLimitSettings responseLimitSettings = new ResponseLimitSettings(clusterSettings, settings); + final RestIndicesAction action = new RestIndicesAction(responseLimitSettings); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(responseLimitSettings); List indicesList = new ArrayList<>(indicesMetadatas.keySet()); // Using half of the indices from metadata list for a page String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); From 5771e811e1a2d8339d897a9ba08689ad3f3efb4a Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 3 Oct 2024 17:10:57 +0530 Subject: [PATCH 181/357] Separate Remote State and Publication enabled and configured methods (#16080) * Separate Remote State and Publication enabled and configured methods Signed-off-by: Shivansh Arora --- .../RemotePublicationConfigurationIT.java | 26 ++++++++++++++ .../remote/RemoteStatePublicationIT.java | 6 +++- .../coordination/CoordinationState.java | 2 +- .../cluster/coordination/Coordinator.java | 8 +++-- .../coordination/JoinTaskExecutor.java | 4 +-- .../PublicationTransportHandler.java | 14 ++------ .../cluster/node/DiscoveryNode.java | 4 +-- .../remote/RemoteClusterStateService.java | 6 ++-- .../index/remote/RemoteIndexPathUploader.java | 4 +-- .../main/java/org/opensearch/node/Node.java | 4 +-- .../remotestore/RemoteStoreNodeAttribute.java | 5 ++- .../test/OpenSearchIntegTestCase.java | 35 +++++++++++-------- 12 files changed, 74 insertions(+), 44 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java index 1d19a4bfd1af5..98859f517aad4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -18,6 +18,7 @@ import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; +import org.junit.Assert; import org.junit.Before; import java.util.Collection; @@ -118,6 +119,31 @@ public Settings.Builder remoteWithRoutingTableNodeSetting() { .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); } + public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); + } + + public void testServiceInitialized_WhenNodeAttributesPresent() { + internalCluster().startClusterManagerOnlyNode( + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + internalCluster().startDataOnlyNodes( + 2, + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); + } + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { internalCluster().startClusterManagerOnlyNode(); internalCluster().startDataOnlyNodes(2); diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 578c922c80a0d..2409b5d0d0e45 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -215,7 +215,11 @@ public void testRemotePublicationDisableIfRemoteStateDisabled() { ensureStableCluster(5); ensureGreen(INDEX_NAME); - assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class)); + RemoteClusterStateService remoteClusterStateService = internalCluster().getCurrentClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + assertFalse(remoteClusterStateService.isRemotePublicationEnabled()); } public void testRemotePublicationDownloadStats() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 9cffc7051d756..e5b07f867d609 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -108,7 +108,7 @@ public CoordinationState( // ToDo: revisit this check while making the setting dynamic this.isRemotePublicationEnabled = isRemoteStateEnabled && REMOTE_PUBLICATION_SETTING.get(settings) - && localNode.isRemoteStatePublicationEnabled(); + && localNode.isRemoteStatePublicationConfigured(); } public boolean isRemotePublicationEnabled() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 9859abe503eaa..1b3ae89251ac0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -189,6 +189,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; private NodeConnectionsService nodeConnectionsService; + private final RemoteClusterStateService remoteClusterStateService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -312,6 +313,7 @@ public Coordinator( this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; this.remoteStoreNodeService = remoteStoreNodeService; + this.remoteClusterStateService = remoteClusterStateService; } private ClusterFormationState getClusterFormationState() { @@ -912,9 +914,9 @@ public DiscoveryStats stats() { stats.add(persistedStateRegistry.getPersistedState(stateType).getStats()); } }); - if (coordinationState.get().isRemotePublicationEnabled()) { - stats.add(publicationHandler.getFullDownloadStats()); - stats.add(publicationHandler.getDiffDownloadStats()); + if (remoteClusterStateService != null) { + stats.add(remoteClusterStateService.getFullDownloadStats()); + stats.add(remoteClusterStateService.getDiffDownloadStats()); } clusterStateStats.setPersistenceStats(stats); return new DiscoveryStats(new PendingClusterStateStats(0, 0, 0), publicationHandler.stats(), clusterStateStats); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 13033b670d44b..aca8653be4dd8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -513,10 +513,10 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi assert existingNodes.isEmpty() == false; Optional remotePublicationNode = existingNodes.stream() - .filter(DiscoveryNode::isRemoteStatePublicationEnabled) + .filter(DiscoveryNode::isRemoteStatePublicationConfigured) .findFirst(); - if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { + if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationConfigured()) { ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index caed2b6eceb49..6277b70f9a7c8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -178,14 +178,6 @@ public PublishClusterStateStats stats() { ); } - public PersistedStateStats getFullDownloadStats() { - return remoteClusterStateService.getFullDownloadStats(); - } - - public PersistedStateStats getDiffDownloadStats() { - return remoteClusterStateService.getDiffDownloadStats(); - } - private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException { try (StreamInput in = CompressedStreamUtils.decompressBytes(request, namedWriteableRegistry)) { ClusterState incomingState; @@ -356,7 +348,7 @@ public PublicationContext newPublicationContext( ) { if (isRemotePublicationEnabled == true) { if (allNodesRemotePublicationEnabled.get() == false) { - if (validateRemotePublicationOnAllNodes(clusterChangedEvent.state().nodes()) == true) { + if (validateRemotePublicationConfiguredOnAllNodes(clusterChangedEvent.state().nodes()) == true) { allNodesRemotePublicationEnabled.set(true); } } @@ -374,11 +366,11 @@ public PublicationContext newPublicationContext( return publicationContext; } - private boolean validateRemotePublicationOnAllNodes(DiscoveryNodes discoveryNodes) { + private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes discoveryNodes) { assert ClusterMetadataManifest.getCodecForVersion(discoveryNodes.getMinNodeVersion()) >= ClusterMetadataManifest.CODEC_V0; for (DiscoveryNode node : discoveryNodes.getNodes().values()) { // if a node is non-remote then created local publication context - if (node.isRemoteStatePublicationEnabled() == false) { + if (node.isRemoteStatePublicationConfigured() == false) { return false; } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index a6f0a457f7f9b..b06c2ef0bdbe4 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -515,10 +515,10 @@ public boolean isRemoteStoreNode() { } /** - * Returns whether remote cluster state publication is enabled on this node + * Returns whether settings required for remote cluster state publication is configured * @return true if the node contains remote cluster state node attribute and remote routing table node attribute */ - public boolean isRemoteStatePublicationEnabled() { + public boolean isRemoteStatePublicationConfigured() { return this.getAttributes() .keySet() .stream() diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ce5e57b79dadb..1d7200792442f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -111,7 +111,7 @@ import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; /** * A Service which provides APIs to upload and download cluster metadata from remote store. @@ -256,7 +256,7 @@ public RemoteClusterStateService( List indexMetadataUploadListeners, NamedWriteableRegistry namedWriteableRegistry ) { - assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) : "Remote cluster state is not configured"; this.nodeId = nodeId; this.repositoriesService = repositoriesService; this.settings = settings; @@ -1061,7 +1061,7 @@ public void close() throws IOException { } public void start() { - assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled"; final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY ); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 5878dff03acc2..2a76a5b966884 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -51,8 +51,8 @@ import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH; import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH; import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategy; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * Uploads the remote store path for all possible combinations of {@link org.opensearch.index.remote.RemoteStoreEnums.DataCategory} @@ -235,7 +235,7 @@ private Repository validateAndGetRepository(String repoSetting) { } public void start() { - assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled"; + assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not configured"; if (isRemoteDataAttributePresent == false) { // If remote store data attributes are not present than we skip this. return; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 56d04df5921ee..fe4bfa1e64ba1 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -311,9 +311,9 @@ import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used @@ -792,7 +792,7 @@ protected Node( final RemoteClusterStateService remoteClusterStateService; final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; final RemoteIndexPathUploader remoteIndexPathUploader; - if (isRemoteStoreClusterStateEnabled(settings)) { + if (isRemoteClusterStateConfigured(settings)) { remoteIndexPathUploader = new RemoteIndexPathUploader( threadPool, settings, diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 55971398634c5..d6a58f8e1d471 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -180,7 +180,7 @@ public static boolean isRemoteDataAttributePresent(Settings settings) { || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; } - public static boolean isRemoteClusterStateAttributePresent(Settings settings) { + public static boolean isRemoteClusterStateConfigured(Settings settings) { return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) .isEmpty() == false; } @@ -194,8 +194,7 @@ public static String getRemoteStoreTranslogRepo(Settings settings) { } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { - return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) - && isRemoteClusterStateAttributePresent(settings); + return RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.get(settings) && isRemoteClusterStateConfigured(settings); } private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 68a2b8086a92e..e27ff311c06f6 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2795,6 +2795,26 @@ public static Settings buildRemoteStoreNodeAttributes( ); } + public static Settings buildRemoteStateNodeAttributes(String stateRepoName, Path stateRepoPath, String stateRepoType) { + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + stateRepoName + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + stateRepoName + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + Settings.Builder settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, stateRepoName) + .put(stateRepoTypeAttributeKey, stateRepoType) + .put(stateRepoSettingsAttributeKeyPrefix + "location", stateRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable); + return settings.build(); + } + private static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, @@ -2850,16 +2870,6 @@ private static Settings buildRemoteStoreNodeAttributes( "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - segmentRepoName - ); - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - segmentRepoName - ); String routingTableRepoAttributeKey = null, routingTableRepoSettingsAttributeKeyPrefix = null; if (routingTableRepoName != null) { routingTableRepoAttributeKey = String.format( @@ -2885,10 +2895,7 @@ private static Settings buildRemoteStoreNodeAttributes( .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put(translogRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable) - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(stateRepoTypeAttributeKey, segmentRepoType) - .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable); + .put(buildRemoteStateNodeAttributes(segmentRepoName, segmentRepoPath, segmentRepoType)); if (routingTableRepoName != null) { settings.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName) .put(routingTableRepoAttributeKey, routingTableRepoType) From 6020c58659c1a7d8d9a65ae449b6c4b411a74d72 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Thu, 3 Oct 2024 17:17:16 +0530 Subject: [PATCH 182/357] [Snapshot V2] Remove orphan timestamps post create snapshot completion (#16148) Signed-off-by: Gaurav Bafna --- .../remotestore/RemoteRestoreSnapshotIT.java | 134 +++++++++++++++--- .../RemoteStorePinnedTimestampsIT.java | 14 +- .../RemoteStorePinnedTimestampService.java | 30 ++++ .../snapshots/SnapshotsService.java | 58 ++++++++ 4 files changed, 216 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index a0183e89bfce2..927dbf9995778 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -9,6 +9,7 @@ package org.opensearch.remotestore; import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; @@ -29,6 +30,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.Index; import org.opensearch.core.rest.RestStatus; @@ -41,6 +43,7 @@ import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryData; @@ -62,8 +65,10 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -73,6 +78,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; +import static org.opensearch.snapshots.SnapshotsService.getPinningEntity; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -753,17 +759,15 @@ public void testInvalidRestoreRequestScenarios() throws Exception { assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); } - public void testCreateSnapshotV2() throws Exception { + public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception { internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); String indexName1 = "testindex1"; String indexName2 = "testindex2"; - String indexName3 = "testindex3"; String snapshotRepoName = "test-create-snapshot-repo"; String snapshotName1 = "test-create-snapshot1"; Path absolutePath1 = randomRepoPath().toAbsolutePath(); - logger.info("Snapshot Path [{}]", absolutePath1); Settings.Builder settings = Settings.builder() .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) @@ -787,27 +791,37 @@ public void testCreateSnapshotV2() throws Exception { indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); + // create an orphan timestamp related to this repo + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + internalCluster().getClusterManagerName() + ); + forceSyncPinnedTimestamps(); + + long pinnedTimestamp = System.currentTimeMillis(); + final CountDownLatch latch = new CountDownLatch(1); + LatchedActionListener latchedActionListener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }, latch); + + remoteStorePinnedTimestampService.pinTimestamp( + pinnedTimestamp, + getPinningEntity(snapshotRepoName, "some_uuid"), + latchedActionListener + ); + latch.await(); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList()); assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); assertThat(snapshotInfo.successfulShards(), greaterThan(0)); assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); - indexDocuments(client, indexName1, 10); - indexDocuments(client, indexName2, 20); - - createIndex(indexName3, indexSettings); - indexDocuments(client, indexName3, 10); - - String snapshotName2 = "test-create-snapshot2"; - - // verify response status if waitForCompletion is not true - RestStatus createSnapshotResponseStatus = client().admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName2) - .get() - .status(); - assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus); + waitUntil(() -> 1 == RemoteStorePinnedTimestampService.getPinnedEntities().size()); } public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception { @@ -879,7 +893,8 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName2)); assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); - + forceSyncPinnedTimestamps(); + assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), 1); } public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException { @@ -955,6 +970,8 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); + forceSyncPinnedTimestamps(); + assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), repositoryData.getSnapshotIds().size()); } public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception { @@ -1017,13 +1034,92 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except logger.info("Exception while creating new-snapshot", e); } + AtomicLong totalSnaps = new AtomicLong(); + // Validate that snapshot is present in repository data assertBusy(() -> { GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName); GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet(); assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1)); + totalSnaps.set(response2.getSnapshots().size()); + }, 30, TimeUnit.SECONDS); thread.join(); + forceSyncPinnedTimestamps(); + waitUntil(() -> { + this.forceSyncPinnedTimestamps(); + return RemoteStorePinnedTimestampService.getPinnedEntities().size() == totalSnaps.intValue(); + }); + } + + public void testCreateSnapshotV2() throws Exception { + internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-create-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + + createRepository(snapshotRepoName, FsRepository.TYPE, settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, Collections.emptyList()); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); + + indexDocuments(client, indexName1, 10); + indexDocuments(client, indexName2, 20); + + createIndex(indexName3, indexSettings); + indexDocuments(client, indexName3, 10); + + String snapshotName2 = "test-create-snapshot2"; + + // verify response status if waitForCompletion is not true + RestStatus createSnapshotResponseStatus = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName2) + .get() + .status(); + assertEquals(RestStatus.ACCEPTED, createSnapshotResponseStatus); + forceSyncPinnedTimestamps(); + assertEquals(2, RemoteStorePinnedTimestampService.getPinnedEntities().size()); + } + + public void forceSyncPinnedTimestamps() { + // for all nodes , run forceSyncPinnedTimestamps() + for (String node : internalCluster().getNodeNames()) { + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + node + ); + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); + } } public void testCreateSnapshotV2WithRedIndex() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index 024e0e952eea5..3e1127e0ce240 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -19,6 +19,9 @@ import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.test.OpenSearchIntegTestCase; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -76,6 +79,11 @@ public void testTimestampPinUnpin() throws Exception { Tuple> pinnedTimestampWithFetchTimestamp_2 = RemoteStorePinnedTimestampService.getPinnedTimestamps(); long lastFetchTimestamp_2 = pinnedTimestampWithFetchTimestamp_2.v1(); assertTrue(lastFetchTimestamp_2 != -1); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + assertEquals(3, pinnedEntities.size()); + assertEquals(Set.of("ss2", "ss3", "ss4"), pinnedEntities.keySet()); + assertEquals(pinnedEntities.get("ss2").size(), 1); + assertEquals(Optional.of(timestamp1).get(), pinnedEntities.get("ss2").get(0)); assertEquals(Set.of(timestamp1, timestamp2, timestamp3), pinnedTimestampWithFetchTimestamp_2.v2()); }); @@ -103,10 +111,14 @@ public void onFailure(Exception e) { // Adding different entity to already pinned timestamp remoteStorePinnedTimestampService.pinTimestamp(timestamp3, "ss5", noOpActionListener); - remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(1)); + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); assertBusy(() -> { Tuple> pinnedTimestampWithFetchTimestamp_3 = RemoteStorePinnedTimestampService.getPinnedTimestamps(); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + assertEquals(3, pinnedEntities.size()); + assertEquals(pinnedEntities.get("ss5").size(), 1); + assertEquals(Optional.of(timestamp3).get(), pinnedEntities.get("ss5").get(0)); long lastFetchTimestamp_3 = pinnedTimestampWithFetchTimestamp_3.v1(); assertTrue(lastFetchTimestamp_3 != -1); assertEquals(Set.of(timestamp1, timestamp3), pinnedTimestampWithFetchTimestamp_3.v2()); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 1448c46583f6a..a3382d8568ec5 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -30,6 +30,9 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,6 +52,7 @@ public class RemoteStorePinnedTimestampService implements Closeable { private static final Logger logger = LogManager.getLogger(RemoteStorePinnedTimestampService.class); private static Tuple> pinnedTimestampsSet = new Tuple<>(-1L, Set.of()); + private static Map> pinnedEntityToTimestampsMap = new HashMap<>(); public static final String PINNED_TIMESTAMPS_PATH_TOKEN = "pinned_timestamps"; public static final String PINNED_TIMESTAMPS_FILENAME_SEPARATOR = "__"; @@ -216,6 +220,16 @@ private long getTimestampFromBlobName(String blobName) { return -1; } + private String getEntityFromBlobName(String blobName) { + String[] blobNameTokens = blobName.split(PINNED_TIMESTAMPS_FILENAME_SEPARATOR); + if (blobNameTokens.length < 2) { + String errorMessage = "Pinned timestamps blob name contains invalid format: " + blobName; + logger.error(errorMessage); + throw new IllegalArgumentException(errorMessage); + } + return String.join(PINNED_TIMESTAMPS_FILENAME_SEPARATOR, Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1)); + } + /** * Unpins a timestamp from the remote store. * @@ -262,6 +276,10 @@ public static Tuple> getPinnedTimestamps() { return pinnedTimestampsSet; } + public static Map> getPinnedEntities() { + return pinnedEntityToTimestampsMap; + } + /** * Inner class for asynchronously updating the pinned timestamp set. */ @@ -283,6 +301,7 @@ protected void runInternal() { Map pinnedTimestampList = blobContainer.listBlobs(); if (pinnedTimestampList.isEmpty()) { pinnedTimestampsSet = new Tuple<>(triggerTimestamp, Set.of()); + pinnedEntityToTimestampsMap = new HashMap<>(); return; } Set pinnedTimestamps = pinnedTimestampList.keySet() @@ -290,8 +309,19 @@ protected void runInternal() { .map(RemoteStorePinnedTimestampService.this::getTimestampFromBlobName) .filter(timestamp -> timestamp != -1) .collect(Collectors.toSet()); + logger.debug("Fetched pinned timestamps from remote store: {} - {}", triggerTimestamp, pinnedTimestamps); pinnedTimestampsSet = new Tuple<>(triggerTimestamp, pinnedTimestamps); + pinnedEntityToTimestampsMap = pinnedTimestampList.keySet() + .stream() + .collect(Collectors.toMap(RemoteStorePinnedTimestampService.this::getEntityFromBlobName, blobName -> { + long timestamp = RemoteStorePinnedTimestampService.this.getTimestampFromBlobName(blobName); + return Collections.singletonList(timestamp); + }, (existingList, newList) -> { + List mergedList = new ArrayList<>(existingList); + mergedList.addAll(newList); + return mergedList; + })); } catch (Throwable t) { logger.error("Exception while fetching pinned timestamp details", t); } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 22b2a72b36026..c80f18cdd82f7 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -38,6 +38,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionRunnable; +import org.opensearch.action.LatchedActionListener; import org.opensearch.action.StepListener; import org.opensearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; @@ -621,6 +622,7 @@ public void onResponse(RepositoryData repositoryData) { return; } listener.onResponse(snapshotInfo); + cleanOrphanTimestamp(repositoryName, repositoryData); } @Override @@ -651,6 +653,57 @@ public TimeValue timeout() { }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } + private void cleanOrphanTimestamp(String repoName, RepositoryData repositoryData) { + Collection snapshotUUIDs = repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); + Map> pinnedEntities = RemoteStorePinnedTimestampService.getPinnedEntities(); + + List orphanPinnedEntities = pinnedEntities.keySet() + .stream() + .filter(pinnedEntity -> isOrphanPinnedEntity(repoName, snapshotUUIDs, pinnedEntity)) + .collect(Collectors.toList()); + + if (orphanPinnedEntities.isEmpty()) { + return; + } + + logger.info("Found {} orphan timestamps. Cleaning it up now", orphanPinnedEntities.size()); + if (tryEnterRepoLoop(repoName)) { + deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities); + leaveRepoLoop(repoName); + } else { + logger.info("Concurrent snapshot create/delete is happening. Skipping clean up of orphan timestamps"); + } + } + + private boolean isOrphanPinnedEntity(String repoName, Collection snapshotUUIDs, String pinnedEntity) { + Tuple tokens = getRepoSnapshotUUIDTuple(pinnedEntity); + return Objects.equals(tokens.v1(), repoName) && snapshotUUIDs.contains(tokens.v2()) == false; + } + + private void deleteOrphanTimestamps(Map> pinnedEntities, List orphanPinnedEntities) { + final CountDownLatch latch = new CountDownLatch(orphanPinnedEntities.size()); + for (String pinnedEntity : orphanPinnedEntities) { + assert pinnedEntities.get(pinnedEntity).size() == 1 : "Multiple timestamps for same repo-snapshot uuid found"; + long orphanTimestamp = pinnedEntities.get(pinnedEntity).get(0); + remoteStorePinnedTimestampService.unpinTimestamp( + orphanTimestamp, + pinnedEntity, + new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) {} + + @Override + public void onFailure(Exception e) {} + }, latch) + ); + } + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private void createSnapshotPreValidations( ClusterState currentState, RepositoryData repositoryData, @@ -707,6 +760,11 @@ public static String getPinningEntity(String repositoryName, String snapshotUUID return repositoryName + SNAPSHOT_PINNED_TIMESTAMP_DELIMITER + snapshotUUID; } + public static Tuple getRepoSnapshotUUIDTuple(String pinningEntity) { + String[] tokens = pinningEntity.split(SNAPSHOT_PINNED_TIMESTAMP_DELIMITER); + return new Tuple<>(tokens[0], tokens[1]); + } + private void cloneSnapshotPinnedTimestamp( RepositoryData repositoryData, SnapshotId sourceSnapshot, From 3d7184b03326d101be737b9a40183a55fb6287c1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 3 Oct 2024 14:29:44 -0400 Subject: [PATCH 183/357] Bump Netty to 4.1.114.Final (#16182) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-buffer-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.112.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.112.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 + 90 files changed, 46 insertions(+), 45 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b69c33df3b6a5..59bcb448bbef2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) +- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 63817289e80c0..8cede6b717883 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -29,7 +29,7 @@ hdrhistogram = 2.2.2 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 -netty = 4.1.112.Final +netty = 4.1.114.Final joda = 2.12.7 # project reactor diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index a42a41b6387c8..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06724b184ee870ecc4d8fc36931beeb3c387b0ee \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..04a48547adb05 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5291a16c10448..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9aed7e78c467d06a47a45b5b27466380a6427e2f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..acbab117e4c15 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +42b1159cac3d196f6bdbd528e29f0fab9dbaae06 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 deleted file mode 100644 index cf50574b87da0..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b23c87a85451b3b0e7c3e8e89698cea6831a8418 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..891ed0a444dc2 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +a01071edffb4812009312b461ce5f160cdec9b75 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index 24e8177190e04..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -375872f1c16bb51aac016ff6ee4f5d28b1288d4d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7df7a05cd7345 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 deleted file mode 100644 index 7c36b789e839c..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d6b2e543749a86957777a46cf68aaa337cc558cb \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..32022e1b2eaa4 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +08134b298d48eec5ddf2e3674a978b52e4667304 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 deleted file mode 100644 index 0196dacfe92ba..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67e590356eb53c20aaabd67f61ae66f628e62e3d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7120cd93e5c0d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f442c794e6fe89e6974f058bf393353e01fb927d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5c26883046fed..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..8cb83fc367d78 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1fd224fdd0b44..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..1be26fee34d46 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index a42a41b6387c8..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -06724b184ee870ecc4d8fc36931beeb3c387b0ee \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..04a48547adb05 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index 22d35128c3ad5..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e683773245716 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d4767d06b22bf..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..2c18924e33c62 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 47af3100f0f2d..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..74ce939dc6190 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8b30272861770..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..c431976b6fbd2 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1a094fa19a623..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..e8080a5b2acb1 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 deleted file mode 100644 index 24e8177190e04..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -375872f1c16bb51aac016ff6ee4f5d28b1288d4d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..7df7a05cd7345 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5fbfde0836e0c..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..fb2d518789a18 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 8dad0e3104dc8..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file From d1fd47c652b4c6a2c0ec5d0ee574a0ff0d263177 Mon Sep 17 00:00:00 2001 From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> Date: Fri, 4 Oct 2024 00:39:19 +0200 Subject: [PATCH 184/357] Implement phone number analyzer (#15915) * add `Strings#isDigits` API inspiration taken from [this SO answer][SO]. note that the stream is not parallelised to avoid the overhead of this as the method is intended to be called primarily with shorter strings where the time to set up would take longer than the actual check. [SO]: https://stackoverflow.com/a/35150400 Signed-off-by: Ralph Ursprung * add `phone` & `phone-search` analyzer + tokenizer this is largely based on [elasticsearch-phone] and internally uses [libphonenumber]. this intentionally only ports a subset of the features: only `phone` and `phone-search` are supported right now, `phone-email` can be added if/when there's a clear need for it. using `libphonenumber` is required since parsing phone numbers is a non-trivial task (even though it might seem trivial at first glance!), as can be seen in the list [falsehoods programmers believe about phone numbers][falsehoods]. this allows defining the region to be used when analysing a phone number. so far only the generic "unkown" region (`ZZ`) had been used which worked as long as international numbers were prefixed with `+` but did not work when using local numbers (e.g. a number stored as `+4158...` was not matched against a number entered as `004158...` or `058...`). example configuration for an index: ```json { "index": { "analysis": { "analyzer": { "phone": { "type": "phone" }, "phone-search": { "type": "phone-search" }, "phone-ch": { "type": "phone", "phone-region": "CH" }, "phone-search-ch": { "type": "phone-search", "phone-region": "CH" } } } } } ``` this creates four analyzers: `phone` and `phone-search` which do not explicitly specify a region and thus fall back to `ZZ` (unknown region, regional version of international dialing prefix (e.g. `00` instead of `+` in most of europe) will not be recognised) and `phone-ch` and `phone-search-ch` which will try to parse the phone number as a swiss phone number (thus e.g. `00` as a prefix is recognised as the international dialing prefix). note that the analyzer is (currently) not meant to find phone numbers in large text documents - instead it should be used on fields which contain just the phone number (though extra text will be ignored) and it collects the whole content of the field into a `String` in memory, making it unsuitable for large field values. this has been implemented in a new plugin which is however part of the central opensearch repository as it was deemed too big an overhead to have it in a separate repository but not important enough to bundle it directly in `analysis-common` (see the discussion on the issue and the PR for further details). note that the new plugin has been added to the exclude list of the javadoc check as this check is overzealous and also complains in many cases where it shouldn't (e.g. on overridden methods - which it should theoretically not do - or constructors which don't even exist). the check first needs to be improved before this exclusion could be removed. closes opensearch-project/OpenSearch#11326 [elasticsearch-phone]: https://github.com/purecloudlabs/elasticsearch-phone [libphonenumber]: https://github.com/google/libphonenumber [falsehoods]: https://github.com/google/libphonenumber/blob/master/FALSEHOODS.md Signed-off-by: Ralph Ursprung --------- Signed-off-by: Ralph Ursprung --- CHANGELOG.md | 1 + gradle/missing-javadoc.gradle | 1 + .../org/opensearch/core/common/Strings.java | 13 + .../opensearch/core/common/StringsTests.java | 11 + plugins/analysis-phonenumber/build.gradle | 21 ++ .../licenses/libphonenumber-8.13.45.jar.sha1 | 1 + .../licenses/libphonenumber-LICENSE.txt | 176 ++++++++++++ .../licenses/libphonenumber-NOTICE.txt | 0 .../phone/PhoneNumberAnalysisPlugin.java | 60 +++++ .../analysis/phone/PhoneNumberAnalyzer.java | 51 ++++ .../phone/PhoneNumberAnalyzerProvider.java | 42 +++ .../phone/PhoneNumberTermTokenizer.java | 157 +++++++++++ .../PhoneNumberTermTokenizerFactory.java | 44 +++ .../phone/PhoneNumberAnalyzerTests.java | 253 ++++++++++++++++++ .../analysis/phone/phone_analysis.json | 22 ++ ...neNumberAnalysisClientYamlTestSuiteIT.java | 49 ++++ .../test/analysis-phone/10_basic.yml | 8 + .../test/analysis-phone/20_search.yml | 56 ++++ 18 files changed, 966 insertions(+) create mode 100644 plugins/analysis-phonenumber/build.gradle create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt create mode 100644 plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java create mode 100644 plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java create mode 100644 plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java create mode 100644 plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml create mode 100644 plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bcb448bbef2..ca23cd59e06d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) +- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index e9a6d798b8323..26898673bf608 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -127,6 +127,7 @@ configure([ project(":plugins:analysis-icu"), project(":plugins:analysis-kuromoji"), project(":plugins:analysis-nori"), + project(":plugins:analysis-phonenumber"), project(":plugins:analysis-phonetic"), project(":plugins:analysis-smartcn"), project(":plugins:analysis-stempel"), diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java index 8fdec670bd9f2..e8379e11ea26a 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java +++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java @@ -815,4 +815,17 @@ public static String toLowercaseAscii(String in) { } return out.toString(); } + + /** + * Check whether every single character in the string is a digit. + * + *

An empty string returns {@code false}.

+ * + * @param s the string, must not be null. + * @return {@code true} if the string only contains digits, {@code false} otherwise. + */ + public static boolean isDigits(final String s) { + return !s.isEmpty() && s.chars().allMatch(Character::isDigit); + } + } diff --git a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java index b79bb6fc89f9e..be7af18b106a8 100644 --- a/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/StringsTests.java @@ -114,4 +114,15 @@ public void testToStringToXContentWithOrWithoutParams() { containsString("\"color_from_param\":\"blue\"") ); } + + public void testIsDigits() { + assertTrue(Strings.isDigits("1")); + assertTrue(Strings.isDigits("123")); + assertFalse(Strings.isDigits("")); + assertFalse(Strings.isDigits("abc")); + assertFalse(Strings.isDigits("123a")); + assertFalse(Strings.isDigits("0x123")); + assertFalse(Strings.isDigits("123.4")); + assertFalse(Strings.isDigits("123f")); + } } diff --git a/plugins/analysis-phonenumber/build.gradle b/plugins/analysis-phonenumber/build.gradle new file mode 100644 index 0000000000000..c9913b36f8508 --- /dev/null +++ b/plugins/analysis-phonenumber/build.gradle @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.yaml-rest-test' + +opensearchplugin { + description 'Adds an analyzer for phone numbers to OpenSearch.' + classname 'org.opensearch.analysis.phone.PhoneNumberAnalysisPlugin' +} + +dependencies { + implementation group: 'com.googlecode.libphonenumber', name: 'libphonenumber', version: '8.13.45' +} diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 new file mode 100644 index 0000000000000..00d393482ee49 --- /dev/null +++ b/plugins/analysis-phonenumber/licenses/libphonenumber-8.13.45.jar.sha1 @@ -0,0 +1 @@ +bfac00f71616796abc7d8b135dda12558a0ccee2 \ No newline at end of file diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt new file mode 100644 index 0000000000000..d9a10c0d8e868 --- /dev/null +++ b/plugins/analysis-phonenumber/licenses/libphonenumber-LICENSE.txt @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt b/plugins/analysis-phonenumber/licenses/libphonenumber-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java new file mode 100644 index 0000000000000..eb12b43f70154 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalysisPlugin.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.opensearch.index.analysis.AnalyzerProvider; +import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.indices.analysis.AnalysisModule; +import org.opensearch.plugins.AnalysisPlugin; +import org.opensearch.plugins.Plugin; + +import java.util.Map; +import java.util.TreeMap; + +/** + * This plugin provides an analyzer and tokenizer for fields which contain phone numbers, supporting a variety of formats + * (with/without international calling code, different country formats, etc.). + */ +public class PhoneNumberAnalysisPlugin extends Plugin implements AnalysisPlugin { + + @Override + public Map>> getAnalyzers() { + Map>> analyzers = new TreeMap<>(); + analyzers.put( + "phone", + (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone", settings, true) + ); + analyzers.put( + "phone-search", + (indexSettings, environment, name, settings) -> new PhoneNumberAnalyzerProvider(indexSettings, "phone-search", settings, false) + ); + return analyzers; + } + + @Override + public Map> getTokenizers() { + Map> tokenizers = new TreeMap<>(); + tokenizers.put( + "phone", + (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory(indexSettings, "phone", settings, true) + ); + tokenizers.put( + "phone-search", + (indexSettings, environment, name, settings) -> new PhoneNumberTermTokenizerFactory( + indexSettings, + "phone-search", + settings, + false + ) + ); + return tokenizers; + } + +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java new file mode 100644 index 0000000000000..cd945e186b2ba --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzer.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.opensearch.common.settings.Settings; + +/** + * Analyzer for phone numbers, using {@link PhoneNumberTermTokenizer}. + * + *

+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers. + * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams) + * for the {@code search_analyzer}. + *

+ * + *

+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that + * phone numbers without the international dialling prefix (using {@code +}) are also tokenized correctly. + *

+ * + *

+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings + * containing phone numbers and not arbitrary text with interspersed phone numbers. + *

+ */ +public class PhoneNumberAnalyzer extends Analyzer { + private final boolean addNgrams; + private final Settings settings; + + /** + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + * @param settings the settings for the analyzer. + */ + public PhoneNumberAnalyzer(final Settings settings, final boolean addNgrams) { + this.addNgrams = addNgrams; + this.settings = settings; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final var tokenizer = new PhoneNumberTermTokenizer(this.settings, this.addNgrams); + return new Analyzer.TokenStreamComponents(tokenizer, tokenizer); + } +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java new file mode 100644 index 0000000000000..272a019ba0f9c --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerProvider.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.AbstractIndexAnalyzerProvider; + +/** + * Provider for {@link PhoneNumberAnalyzer}. + */ +public class PhoneNumberAnalyzerProvider extends AbstractIndexAnalyzerProvider { + + private final PhoneNumberAnalyzer analyzer; + + /** + * @param indexSettings the settings of the index. + * @param name the analyzer name. + * @param settings the settings for the analyzer. + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + */ + public PhoneNumberAnalyzerProvider( + final IndexSettings indexSettings, + final String name, + final Settings settings, + final boolean addNgrams + ) { + super(indexSettings, name, settings); + this.analyzer = new PhoneNumberAnalyzer(settings, addNgrams); + } + + @Override + public PhoneNumberAnalyzer get() { + return this.analyzer; + } +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java new file mode 100644 index 0000000000000..6b95594204eb4 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizer.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import com.google.i18n.phonenumbers.NumberParseException; +import com.google.i18n.phonenumbers.PhoneNumberUtil; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.opensearch.common.io.Streams; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Optional; +import java.util.Set; + +/** + * This tokenizes a phone number into its individual parts, using {@link PhoneNumberUtil}. + * + *

+ * You can use the {@code phone} and {@code phone-search} analyzers on your fields to index phone numbers. + * Use {@code phone} (which creates ngrams) for the {@code analyzer} and {@code phone-search} (which doesn't create ngrams) + * for the {@code search_analyzer}. + *

+ * + *

+ * You optionally can specify a region with the {@code phone-region} setting for the phone number which will ensure that + * phone numbers without the international dialling prefix (using {@code +}) are also tokenized correctly. + *

+ * + *

+ * Note that the tokens will not refer to a specific position in the stream as the tokenizer is expected to be used on strings + * containing phone numbers and not arbitrary text with interspersed phone numbers. + *

+ */ +public final class PhoneNumberTermTokenizer extends Tokenizer { + private final boolean addNgrams; + private final Settings settings; + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private Iterator tokenIterator; + + /** + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + * @param settings the settings for the analyzer. + */ + public PhoneNumberTermTokenizer(final Settings settings, final boolean addNgrams) { + super(); + this.addNgrams = addNgrams; + this.settings = settings; + } + + /** {@inheritDoc} */ + @Override + public void reset() throws IOException { + super.reset(); + tokenIterator = null; + } + + /** {@inheritDoc} */ + @Override + public boolean incrementToken() throws IOException { + clearAttributes(); + if (tokenIterator == null) { + tokenIterator = getTokens().iterator(); + } + if (tokenIterator.hasNext()) { + termAtt.append(tokenIterator.next()); + return true; + } + return false; + } + + /** + * Search for a phone number in the input and tokenize it. + * + *

+ * The tokens include the full phone number with and without country prefix (if it could be identified) and - if + * enabled by {@link #addNgrams} - an ngram of the phone number. + *

+ * + * @return all tokens (unique, unordered). + * @throws IOException in case the input cannot be read. + */ + private Set getTokens() throws IOException { + final var tokens = new HashSet(); + + var input = Streams.copyToString(this.input); + + tokens.add(input); + + // Rip off the "tel:" or "sip:" prefix + if (input.indexOf("tel:") == 0 || input.indexOf("sip:") == 0) { + tokens.add(input.substring(0, 4)); + input = input.substring(4); + } + + final var startIndex = input.startsWith("+") ? 1 : 0; + // Add the complete input but skip a leading + + tokens.add(input.substring(startIndex)); + + // Drop anything after @. Most likely there's nothing of interest + final var posAt = input.indexOf('@'); + if (posAt != -1) { + input = input.substring(0, posAt); + + // Add a token for the raw unmanipulated address. Note this could be a username (sip) instead of telephone + // number so take it as is + tokens.add(input.substring(startIndex)); + } + + // Let google's libphone try to parse it + final var phoneUtil = PhoneNumberUtil.getInstance(); + Optional countryCode = Optional.empty(); + try { + // ZZ is the generic "I don't know the country code" region. Google's libphone library will try to infer it. + final var region = this.settings.get("phone-region", "ZZ"); + final var numberProto = phoneUtil.parse(input, region); + if (numberProto != null) { + // Libphone likes it! + countryCode = Optional.of(String.valueOf(numberProto.getCountryCode())); + input = String.valueOf(numberProto.getNationalNumber()); + + // Add Country code, extension, and the number as tokens + tokens.add(countryCode.get()); + tokens.add(countryCode.get() + input); + if (!Strings.isEmpty(numberProto.getExtension())) { + tokens.add(numberProto.getExtension()); + } + + tokens.add(input); + } + } catch (final NumberParseException | StringIndexOutOfBoundsException e) { + // Libphone didn't like it, no biggie. We'll just ngram the number as it is. + } + + // ngram the phone number, e.g. 19198243333 produces 9, 91, 919, etc + if (this.addNgrams && Strings.isDigits(input)) { + for (int count = 1; count <= input.length(); ++count) { + final var token = input.substring(0, count); + tokens.add(token); + // If there was a country code, add more ngrams such that 19198243333 produces 19, 191, 1919, etc + countryCode.ifPresent(s -> tokens.add(s + token)); + } + } + + return tokens; + } + +} diff --git a/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java new file mode 100644 index 0000000000000..fde44e15c9667 --- /dev/null +++ b/plugins/analysis-phonenumber/src/main/java/org/opensearch/analysis/phone/PhoneNumberTermTokenizerFactory.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Tokenizer; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.AbstractTokenizerFactory; + +/** + * Factory for {@link PhoneNumberTermTokenizer}. + */ +public class PhoneNumberTermTokenizerFactory extends AbstractTokenizerFactory { + private final Settings settings; + private final boolean addNgrams; + + /** + * @param indexSettings the settings of the index. + * @param name the tokenizer name. + * @param settings the settings for the analyzer. + * @param addNgrams defines whether ngrams for the phone number should be added. Set to true for indexing and false for search. + */ + public PhoneNumberTermTokenizerFactory( + final IndexSettings indexSettings, + final String name, + final Settings settings, + final boolean addNgrams + ) { + super(indexSettings, settings, name); + this.settings = settings; + this.addNgrams = addNgrams; + } + + @Override + public Tokenizer create() { + return new PhoneNumberTermTokenizer(this.settings, this.addNgrams); + } +} diff --git a/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java new file mode 100644 index 0000000000000..332f6d21f47d6 --- /dev/null +++ b/plugins/analysis-phonenumber/src/test/java/org/opensearch/analysis/phone/PhoneNumberAnalyzerTests.java @@ -0,0 +1,253 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analysis.phone; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.opensearch.index.analysis.AnalysisTestsHelper; +import org.opensearch.test.OpenSearchTokenStreamTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.hasItemInArray; + +public class PhoneNumberAnalyzerTests extends OpenSearchTokenStreamTestCase { + private static final String RESOURCE = "/org/opensearch/analysis/phone/phone_analysis.json"; + + private static Analyzer phoneAnalyzer; + private static Analyzer phoneSearchAnalyzer; + private static Analyzer phoneCHAnalyzer; + private static Analyzer phoneSearchCHAnalyzer; + + @BeforeClass + public static void beforeClass() throws IOException { + final var analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath( + createTempDir(), + RESOURCE, + new PhoneNumberAnalysisPlugin() + ); + phoneAnalyzer = analysis.indexAnalyzers.get("phone"); + assertNotNull(phoneAnalyzer); + phoneSearchAnalyzer = analysis.indexAnalyzers.get("phone-search"); + assertNotNull(phoneSearchAnalyzer); + phoneCHAnalyzer = analysis.indexAnalyzers.get("phone-ch"); + assertNotNull(phoneCHAnalyzer); + phoneSearchCHAnalyzer = analysis.indexAnalyzers.get("phone-search-ch"); + assertNotNull(phoneSearchCHAnalyzer); + } + + /** + * Test for all tokens which are emitted by the "phone" analyzer. + */ + public void testEuropeDetailled() throws IOException { + assertTokensAreInAnyOrder( + phoneAnalyzer, + "tel:+441344840400", + Arrays.asList( + "tel:+441344840400", + "tel:", + "441344840400", + "44", + "1344840400", + "1", + "441", + "13", + "4413", + "134", + "44134", + "1344", + "441344", + "13448", + "4413448", + "134484", + "44134484", + "1344840", + "441344840", + "13448404", + "4413448404", + "134484040", + "44134484040" + ) + ); + } + + /** + * Test for all tokens which are emitted by the "phone" analyzer. + */ + public void testEuropeDetailledSearch() throws IOException { + assertTokensAreInAnyOrder( + phoneSearchAnalyzer, + "tel:+441344840400", + Arrays.asList("tel:+441344840400", "tel:", "441344840400", "44", "1344840400") + ); + } + + public void testEurope() throws IOException { + assertTokensInclude("tel:+441344840400", Arrays.asList("44", "1344", "1344840400", "441344840400")); + } + + public void testGermanCastle() throws IOException { + assertTokensInclude("tel:+498362930830", Arrays.asList("49", "498362930830", "8362930830")); + } + + public void testBMWofSydney() throws IOException { + assertTokensInclude("tel:+61293344555", Arrays.asList("61", "293344555", "61293344555")); + } + + public void testCoffeeShopInIreland() throws IOException { + assertTokensInclude("tel:+442890319416", Arrays.asList("44", "289", "2890319416", "442890319416")); + } + + public void testTelWithCountryCode() throws IOException { + assertTokensInclude("tel:+17177158163", Arrays.asList("1", "717", "7177", "17177158163")); + } + + public void testTelWithCountryCode2() throws IOException { + assertTokensInclude("tel:+12177148350", Arrays.asList("1", "217", "2177", "2177148350", "12177148350")); + } + + public void testNewTollFreeNumber() throws IOException { + assertTokensInclude("tel:+18337148350", Arrays.asList("1", "833", "8337", "8337148350", "18337148350")); + } + + public void testMissingCountryCode() throws IOException { + assertTokensInclude("tel:8177148350", Arrays.asList("817", "8177", "81771", "817714", "8177148350")); + } + + public void testSipWithNumericUsername() throws IOException { + assertTokensInclude("sip:222@autosbcpc", Arrays.asList("222")); + } + + public void testTruncatedNumber() throws IOException { + assertTokensInclude("tel:5551234", Arrays.asList("5551234")); + } + + public void testSipWithAlphabeticUsername() throws IOException { + assertTokensInclude("sip:abc@autosbcpc", Arrays.asList("abc")); + } + + public void testGarbageInGarbageOut() throws IOException { + assertTokensInclude("test", Arrays.asList("test")); + } + + public void testSipWithCountryCode() throws IOException { + assertTokensInclude("sip:+14177141363@178.97.105.13;isup-oli=0;pstn-params=808481808882", Arrays.asList("417", "4177", "14177")); + } + + public void testSipWithTelephoneExtension() throws IOException { + assertTokensInclude("sip:+13169410766;ext=2233@178.17.10.117:8060", Arrays.asList("316", "2233", "1316")); + } + + public void testSipWithUsername() throws IOException { + assertTokensInclude("sip:JeffSIP@178.12.220.18", Arrays.asList("JeffSIP")); + } + + public void testPhoneNumberWithoutPrefix() throws IOException { + assertTokensInclude("+14177141363", Arrays.asList("14177141363", "417", "4177", "14177")); + } + + public void testSipWithoutDomainPart() throws IOException { + assertTokensInclude("sip:+122882", Arrays.asList("122882", "122", "228", "1228", "2288", "12288")); + } + + public void testTelPrefix() throws IOException { + assertTokensInclude("tel:+1228", Arrays.asList("1228", "122", "228")); + } + + public void testNumberPrefix() throws IOException { + assertTokensInclude("+1228", Arrays.asList("1228", "122", "228")); + } + + public void testInternationalPrefixWithZZ() throws IOException { + assertTokensInclude(phoneAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testInternationalPrefixWithCH() throws IOException { + assertTokensInclude(phoneCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testNationalPrefixWithCH() throws IOException { + // + is equivalent to 00 in Switzerland + assertTokensInclude(phoneCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testLocalNumberWithCH() throws IOException { + // when omitting the international prefix swiss numbers must start with '0' + assertTokensInclude(phoneCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchInternationalPrefixWithZZ() throws IOException { + assertTokensInclude(phoneSearchAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchInternationalPrefixWithCH() throws IOException { + assertTokensInclude(phoneSearchCHAnalyzer, "+41583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchNationalPrefixWithCH() throws IOException { + // + is equivalent to 00 in Switzerland + assertTokensInclude(phoneSearchCHAnalyzer, "0041583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + public void testSearchLocalNumberWithCH() throws IOException { + // when omitting the international prefix swiss numbers must start with '0' + assertTokensInclude(phoneSearchCHAnalyzer, "0583161010", Arrays.asList("41", "41583161010", "583161010")); + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for order. Use {@link #assertTokensInclude} if completeness is not needed. + */ + private void assertTokensAreInAnyOrder(final Analyzer analyzer, final String input, final List expectedTokens) + throws IOException { + final var ts = analyzer.tokenStream("test", input); + final var allTokens = getAllTokens(ts).toArray(); + assertThat(allTokens, arrayContainingInAnyOrder(expectedTokens.toArray())); + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for completeness or order. + */ + private void assertTokensInclude(final Analyzer analyzer, final String input, final List expectedTokens) throws IOException { + final var ts = analyzer.tokenStream("test", input); + final var allTokens = getAllTokens(ts).toArray(); + for (final var expectedToken : expectedTokens) { + assertThat(allTokens, hasItemInArray(expectedToken)); + } + } + + /** + * Unlike {@link #assertTokenStreamContents(TokenStream, String[])} this only asserts whether the generated tokens + * contain the required ones but does not check for completeness or order. + * This uses {@link #phoneAnalyzer}. + */ + private void assertTokensInclude(final String input, final List expectedTokens) throws IOException { + this.assertTokensInclude(phoneAnalyzer, input, expectedTokens); + } + + private List getAllTokens(final TokenStream ts) throws IOException { + final var tokens = new ArrayList(); + final var termAtt = ts.getAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + tokens.add(termAtt.toString()); + } + ts.end(); + ts.close(); + return tokens; + } + +} diff --git a/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json new file mode 100644 index 0000000000000..7e45177c57492 --- /dev/null +++ b/plugins/analysis-phonenumber/src/test/resources/org/opensearch/analysis/phone/phone_analysis.json @@ -0,0 +1,22 @@ +{ + "index": { + "analysis": { + "analyzer": { + "phone": { + "type": "phone" + }, + "phone-search": { + "type": "phone-search" + }, + "phone-ch": { + "type": "phone", + "phone-region": "CH" + }, + "phone-search-ch": { + "type": "phone-search", + "phone-region": "CH" + } + } + } + } +} diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..d514a3329a1a7 --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/java/org/opensearch/analysis/phone/PhoneNumberAnalysisClientYamlTestSuiteIT.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.analysis.phone; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class PhoneNumberAnalysisClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + public PhoneNumberAnalysisClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml new file mode 100644 index 0000000000000..5bea0cf4650d6 --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/10_basic.yml @@ -0,0 +1,8 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^analysis-phonenumber\n$/ diff --git a/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml new file mode 100644 index 0000000000000..0bd7d2c371bfc --- /dev/null +++ b/plugins/analysis-phonenumber/src/yamlRestTest/resources/rest-api-spec/test/analysis-phone/20_search.yml @@ -0,0 +1,56 @@ +# Integration tests for phone analysis components +# +--- +"Index phone number content": + - do: + indices.create: + index: test + body: + settings: + index: + analysis: + analyzer: + phone-ch: + type: "phone" + "phone-region": "CH" + phone-search-ch: + type: "phone-search" + "phone-region": "CH" + mappings: + properties: + phone: + type: text + analyzer: "phone" + search_analyzer: "phone-search" + phone-ch: + type: text + analyzer: "phone-ch" + search_analyzer: "phone-search-ch" + + - do: + index: + index: test + id: 1 + body: { "phone": "+41 58 316 10 10", "phone-ch": "058 316 10 10" } + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match: + "phone": "+41583161010" + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + match: + "phone-ch": "+41583161010" + - match: { hits.total: 1 } From f9e0c85c07cf9aa043e714b610d87dcb9bc9b163 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 4 Oct 2024 00:54:50 -0400 Subject: [PATCH 185/357] Fix japicmp configuration by treating abstract-to-default method changes as non-breaking (#16181) Signed-off-by: Andriy Redko --- server/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/build.gradle b/server/build.gradle index 83a04ef12d13b..eccaf8a127647 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -407,6 +407,8 @@ tasks.named("sourcesJar").configure { /** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */ tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}") + // See please https://github.com/siom79/japicmp/issues/201 + compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT" ] oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar")) newClasspath.from(tasks.named('jar')) onlyModified = true From aef7ecace31a4ff3986bb287fec42f3fcfb30bad Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 4 Oct 2024 10:50:53 +0530 Subject: [PATCH 186/357] Add more unit tests for RemoteStoreUtils and RemoteFsTimestampAwareTranslog (#16151) Signed-off-by: Sachin Kale --- .../index/remote/RemoteStoreUtils.java | 1 + .../RemoteFsTimestampAwareTranslog.java | 15 +- .../index/remote/RemoteStoreUtilsTests.java | 97 +++++++ .../RemoteFsTimestampAwareTranslogTests.java | 249 +++++++++++++++++- 4 files changed, 353 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java index edcfe17d31fb8..32a1ca0e5d5ab 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java @@ -477,6 +477,7 @@ private static Set getPinnedTimestampLockedFiles( for (Long pinnedTimestamp : pinnedTimestampSet) { String cachedFile = metadataFilePinnedTimestampMap.get(pinnedTimestamp); if (cachedFile != null) { + assert metadataFiles.contains(cachedFile) : "Metadata files should contain [" + cachedFile + "]"; implicitLockedFiles.add(cachedFile); } else { newPinnedTimestamps.add(pinnedTimestamp); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index ede422ea3c4f7..3ccacde22bbfc 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -233,6 +233,7 @@ public void onResponse(List blobMetadata) { // Update cache to keep only those metadata files that are not getting deleted oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); + oldFormatMetadataFilePrimaryTermMap.keySet().retainAll(metadataFilesNotToBeDeleted); // Delete stale primary terms deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted); } else { @@ -408,9 +409,9 @@ protected Tuple getMinMaxTranslogGenerationFromMetadataFile( } } - private void deleteStaleRemotePrimaryTerms(List metadataFiles) { + private void deleteStaleRemotePrimaryTerms(List metadataFilesNotToBeDeleted) { deleteStaleRemotePrimaryTerms( - metadataFiles, + metadataFilesNotToBeDeleted, translogTransferManager, oldFormatMetadataFilePrimaryTermMap, minPrimaryTermInRemote, @@ -425,7 +426,7 @@ private void deleteStaleRemotePrimaryTerms(List metadataFiles) { * This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator. */ protected static void deleteStaleRemotePrimaryTerms( - List metadataFiles, + List metadataFilesNotToBeDeleted, TranslogTransferManager translogTransferManager, Map> oldFormatMetadataFilePrimaryTermMap, AtomicLong minPrimaryTermInRemoteAtomicLong, @@ -434,15 +435,15 @@ protected static void deleteStaleRemotePrimaryTerms( // The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there // are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part // of older primary term. - if (metadataFiles.isEmpty()) { + if (metadataFilesNotToBeDeleted.isEmpty()) { logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms"); return; } - Optional minPrimaryTermFromMetadataFiles = metadataFiles.stream().map(file -> { + Optional minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> { try { return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1(); } catch (IOException e) { - return Long.MAX_VALUE; + return Long.MIN_VALUE; } }).min(Long::compareTo); // First we delete all stale primary terms folders from remote store @@ -459,7 +460,7 @@ protected static void deleteStaleRemotePrimaryTerms( } } - private static Long getMinPrimaryTermInRemote( + protected static Long getMinPrimaryTermInRemote( AtomicLong minPrimaryTermInRemote, TranslogTransferManager translogTransferManager, Logger logger diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java index be30de97ee830..d134a2bad7b74 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java @@ -1083,4 +1083,101 @@ public void testIsPinnedTimestampStateStaleFeatureEnabled() { setupRemotePinnedTimestampFeature(true); assertTrue(RemoteStoreUtils.isPinnedTimestampStateStale()); } + + public void testGetPinnedTimestampLockedFilesWithCache() { + setupRemotePinnedTimestampFeature(true); + + Map metadataFilePinnedTimestampCache = new HashMap<>(); + + // Pinned timestamps 800, 900, 1000, 2000 + // Metadata with timestamp 990, 995, 1000, 1001 + // Metadata timestamp 1000 <= Pinned Timestamp 1000 + // Metadata timestamp 1001 <= Pinned Timestamp 2000 + Tuple, Set> metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L), + Set.of(800L, 900L, 1000L, 2000L), + metadataFilePinnedTimestampCache + ); + Map metadataFiles = metadataAndLocks.v1(); + Set implicitLockedFiles = metadataAndLocks.v2(); + + assertEquals(2, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1001L))); + // Now we cache all the matches except the last one. + assertEquals(1, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L), + Set.of(800L, 900L, 1000L, 2000L, 3000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(3, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(1000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2200L))); + assertEquals(2, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(1000L), metadataFilePinnedTimestampCache.get(1000L)); + assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(990L, 995L, 1000L, 1001L, 2000L, 2200L, 2500L), + Set.of(2000L, 3000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(2, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2000L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L))); + assertEquals(1, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(2000L), metadataFilePinnedTimestampCache.get(2000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ); + metadataFiles = metadataAndLocks.v1(); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(4, implicitLockedFiles.size()); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(2500L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(3001L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(4600L))); + assertTrue(implicitLockedFiles.contains(metadataFiles.get(5010L))); + assertEquals(3, metadataFilePinnedTimestampCache.size()); + assertEquals(metadataFiles.get(2500L), metadataFilePinnedTimestampCache.get(3000L)); + assertEquals(metadataFiles.get(3001L), metadataFilePinnedTimestampCache.get(4000L)); + assertEquals(metadataFiles.get(4600L), metadataFilePinnedTimestampCache.get(5000L)); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(0, implicitLockedFiles.size()); + assertEquals(3, metadataFilePinnedTimestampCache.size()); + + assertThrows( + AssertionError.class, + () -> testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 3001L, 4200L, 4600L, 5010L), + Set.of(3000L, 4000L, 5000L, 6000L), + metadataFilePinnedTimestampCache + ) + ); + + metadataAndLocks = testGetPinnedTimestampLockedFilesWithPinnedTimestamps( + List.of(2000L, 2200L, 2500L, 3001L, 4200L, 4600L, 5010L), + Set.of(), + metadataFilePinnedTimestampCache + ); + implicitLockedFiles = metadataAndLocks.v2(); + assertEquals(0, implicitLockedFiles.size()); + assertEquals(0, metadataFilePinnedTimestampCache.size()); + } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index e6871414cf5e0..73db3314f4d1e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -721,6 +721,42 @@ public void testGetGenerationsToBeDeleted() throws IOException { assertTrue(generations.isEmpty()); } + public void testGetGenerationsToBeDeletedWithGenerationInRemote() throws IOException { + List metadataFilesNotToBeDeleted = List.of( + // 1 to 4 + "metadata__9223372036854775806__9223372036854775803__9223370311919910398__31__9223372036854775806__1__1", + // 26 to 30 + "metadata__9223372036854775806__9223372036854775777__9223370311919910398__31__9223372036854775781__1__1", + // 42 to 100 + "metadata__9223372036854775806__9223372036854775707__9223370311919910403__31__9223372036854775765__1__1" + ); + List metadataFilesToBeDeleted = List.of( + // 4 to 7 + "metadata__9223372036854775806__9223372036854775800__9223370311919910398__31__9223372036854775803__1__1", + // 17 to 37 + "metadata__9223372036854775806__9223372036854775770__9223370311919910398__31__9223372036854775790__1__1", + // 27 to 42 + "metadata__9223372036854775806__9223372036854775765__9223370311919910403__31__9223372036854775780__1__1" + ); + Set generations = ((RemoteFsTimestampAwareTranslog) translog).getGenerationsToBeDeleted( + metadataFilesNotToBeDeleted, + metadataFilesToBeDeleted, + 35 + ); + Set md1Generations = LongStream.rangeClosed(5, 7).boxed().collect(Collectors.toSet()); + Set md2Generations = LongStream.rangeClosed(17, 25).boxed().collect(Collectors.toSet()); + Set md3Generations = LongStream.rangeClosed(31, 34).boxed().collect(Collectors.toSet()); + + assertTrue(generations.containsAll(md1Generations)); + assertTrue(generations.containsAll(md2Generations)); + assertTrue(generations.containsAll(md3Generations)); + + generations.removeAll(md1Generations); + generations.removeAll(md2Generations); + generations.removeAll(md3Generations); + assertTrue(generations.isEmpty()); + } + public void testGetMetadataFilesToBeDeletedNoExclusion() { updatePinnedTimstampTask.run(); @@ -982,6 +1018,16 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException translogTransferManager ) ); + // Calling it again to check if the details are getting fetched from the cache. + // Number of calls to readMetadata will be able to verify this. + assertEquals( + new Tuple<>(701L, 1008L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1", + translogTransferManager + ) + ); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); when(md2.getMinTranslogGeneration()).thenReturn(4L); when(md2.getGeneration()).thenReturn(7L); @@ -994,9 +1040,81 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException translogTransferManager ) ); + // Calling it again to check if the details are getting fetched from the cache. + // Number of calls to readMetadata will be able to verify this. + assertEquals( + new Tuple<>(4L, 7L), + translog.getMinMaxTranslogGenerationFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", + translogTransferManager + ) + ); + + verify(translogTransferManager, times(1)).readMetadata( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1" + ); + verify(translogTransferManager, times(1)).readMetadata( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1" + ); + } + + public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog; + + // Fetch generations directly from the filename + assertEquals( + new Tuple<>(1L, 1008L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854774799__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(4L, 7L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775800__9223372036854775800__9223370311919910398__31__9223372036854775803__4__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(10L, 10L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775797__9223372036854775800__9223370311919910398__31__9223372036854775803__10__1", + translogTransferManager, + new HashMap<>() + ) + ); + + // For older md filenames, it needs to read the content + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "1", "23", "1", "34", "2")); + when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1")) + .thenReturn(md1); + assertEquals( + new Tuple<>(1L, 2L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1", + translogTransferManager, + new HashMap<>() + ) + ); + assertEquals( + new Tuple<>(4L, 7L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", + translogTransferManager, + Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L)) + ) + ); - verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__1"); - verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"); + verify(translogTransferManager).readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1"); + verify(translogTransferManager, times(0)).readMetadata( + "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1" + ); } public void testDeleteStaleRemotePrimaryTerms() throws IOException { @@ -1035,6 +1153,96 @@ public void testDeleteStaleRemotePrimaryTerms() throws IOException { verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong()); } + public void testDeleteStaleRemotePrimaryTermsOldFormat() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L)); + + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5")); + when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md1); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); + when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3")); + when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md2); + TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class); + when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2")); + when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md3); + + AtomicLong minPrimaryTermInRemote = new AtomicLong(Long.MAX_VALUE); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager).deletePrimaryTermsAsync(2L); + assertEquals(2, minPrimaryTermInRemote.get()); + + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + // This means there are no new invocations of deletePrimaryTermAsync + verify(translogTransferManager, times(1)).deletePrimaryTermsAsync(anyLong()); + } + + public void testDeleteStaleRemotePrimaryTermsOldFormatException() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + + List metadataFiles = List.of( + // PT 4 to 9 + "metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 7 + "metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1", + // PT 2 to 6 + "metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1" + ); + + Logger staticLogger = LogManager.getLogger(RemoteFsTimestampAwareTranslogTests.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(1L, 2L, 3L, 4L)); + + TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "4", "23", "5", "34", "5")); + when(translogTransferManager.readMetadata("metadata__9223372036854775798__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md1); + TranslogTransferMetadata md2 = mock(TranslogTransferMetadata.class); + when(md2.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "3")); + when(translogTransferManager.readMetadata("metadata__9223372036854775800__9223372036854774799__9223370311919910393__node1__1")) + .thenReturn(md2); + TranslogTransferMetadata md3 = mock(TranslogTransferMetadata.class); + when(md3.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "2", "23", "2", "34", "2")); + // Exception while reading this file + when(translogTransferManager.readMetadata("metadata__9223372036854775801__9223372036854774799__9223370311919910393__node1__1")) + .thenThrow(new IOException()); + + AtomicLong minPrimaryTermInRemote = new AtomicLong(4); + RemoteFsTimestampAwareTranslog.deleteStaleRemotePrimaryTerms( + metadataFiles, + translogTransferManager, + new HashMap<>(), + minPrimaryTermInRemote, + staticLogger + ); + verify(translogTransferManager, times(0)).deletePrimaryTermsAsync(anyLong()); + } + public void testDeleteStaleRemotePrimaryTermsNoPrimaryTermInRemote() throws IOException { TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); @@ -1087,4 +1295,41 @@ public void testDeleteStaleRemotePrimaryTermsPrimaryTermInRemoteIsBigger() throw assertEquals(2, minPrimaryTermInRemote.get()); } + public void testGetMinPrimaryTermInRemoteAlreadyFetched() { + Long expectedMinPrimaryTerm = 12L; + assertEquals(expectedMinPrimaryTerm, RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(12), null, null)); + } + + public void testGetMinPrimaryTermInRemoteNotFetchedEmpty() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of()); + Long expectedMinPrimaryTerm = Long.MAX_VALUE; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, null) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } + + public void testGetMinPrimaryTermInRemoteNotFetchedException() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenThrow(new IOException()); + Long expectedMinPrimaryTerm = Long.MAX_VALUE; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } + + public void testGetMinPrimaryTermInRemoteNotFetched() throws IOException { + TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); + when(translogTransferManager.listPrimaryTermsInRemote()).thenReturn(Set.of(12L, 23L, 34L, 4L, 89L)); + Long expectedMinPrimaryTerm = 4L; + assertEquals( + expectedMinPrimaryTerm, + RemoteFsTimestampAwareTranslog.getMinPrimaryTermInRemote(new AtomicLong(Long.MAX_VALUE), translogTransferManager, logger) + ); + verify(translogTransferManager).listPrimaryTermsInRemote(); + } } From ba8f1bec392d5f4301b9e043d06456eefd89e8fd Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:02:10 +0530 Subject: [PATCH 187/357] Run queued operations post v2 operations completion (#16179) Signed-off-by: Gaurav Bafna --- .../snapshots/ConcurrentSnapshotsV2IT.java | 118 ++++++++++++++++++ .../snapshots/SnapshotsService.java | 24 ++-- 2 files changed, 133 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index f20fddb6af26c..97d63d0e2380a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -479,6 +479,124 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception { assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); } + public void testDeleteWhileV2CreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + + // Creating a v2 repo + settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture a = startDeleteSnapshot(repoName, "snapshot-v1"); + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); + assertTrue(a.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(1, snapInfo.size()); + assertThat(snapInfo, contains(csr.getSnapshotInfo())); + } + + public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + startFullSnapshot(repoName, "snapshot-v1-2").actionGet(); + + // Creating a v2 repo + settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture startDeleteSnapshot = startDeleteSnapshot(repoName, "snapshot-v1"); + ActionFuture startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1-2", "snapshot-v1-2-clone"); + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); + assertTrue(startDeleteSnapshot.actionGet().isAcknowledged()); + assertTrue(startCloneSnapshot.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(3, snapInfo.size()); + } + protected ActionFuture startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) { logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName); return clusterAdmin().prepareCloneSnapshot(repoName, sourceSnapshotName, snapshotName).setIndices("*").execute(); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index c80f18cdd82f7..541a87c200883 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -382,7 +382,7 @@ public ClusterState execute(ClusterState currentState) { ); if (request.partial() == false) { Set missing = new HashSet<>(); - for (final Map.Entry entry : shards.entrySet()) { + for (final Map.Entry entry : shards.entrySet()) { if (entry.getValue().state() == ShardState.MISSING) { missing.add(entry.getKey().getIndex().getName()); } @@ -606,8 +606,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl new ActionListener() { @Override public void onResponse(RepositoryData repositoryData) { - leaveRepoLoop(repositoryName); if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { + leaveRepoLoop(repositoryName); failSnapshotCompletionListeners( snapshot, new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") @@ -622,6 +622,9 @@ public void onResponse(RepositoryData repositoryData) { return; } listener.onResponse(snapshotInfo); + // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations + // can get queued . This is triggering them. + runNextQueuedOperation(repositoryData, repositoryName, true); cleanOrphanTimestamp(repositoryName, repositoryData); } @@ -1010,8 +1013,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl new ActionListener() { @Override public void onResponse(RepositoryData repositoryData) { - leaveRepoLoop(repositoryName); if (!clusterService.state().nodes().isLocalNodeElectedClusterManager()) { + leaveRepoLoop(repositoryName); failSnapshotCompletionListeners( snapshot, new SnapshotException(snapshot, "Aborting Snapshot-v2 clone, no longer cluster manager") @@ -1027,6 +1030,9 @@ public void onResponse(RepositoryData repositoryData) { } logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); listener.onResponse(null); + // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations + // can get queued . This is triggering them. + runNextQueuedOperation(repositoryData, repositoryName, true); } @Override @@ -1740,7 +1746,7 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) { /** * Updates the state of in-progress snapshots in reaction to a change in the configuration of the cluster nodes (cluster-manager fail-over or * disconnect of a data node that was executing a snapshot) or a routing change that started shards whose snapshot state is - * {@link SnapshotsInProgress.ShardState#WAITING}. + * {@link ShardState#WAITING}. * * @param changedNodes true iff either a cluster-manager fail-over occurred or a data node that was doing snapshot work got removed from the * cluster @@ -2645,7 +2651,7 @@ private static List matchingSnapshotIds( } } } - return Collections.unmodifiableList(new ArrayList<>(foundSnapshots)); + return unmodifiableList(new ArrayList<>(foundSnapshots)); } // Return in-progress snapshot entries by name and repository in the given cluster state or null if none is found @@ -2795,7 +2801,7 @@ public ClusterState execute(ClusterState currentState) { reusedExistingDelete = true; return currentState; } - final List toDelete = Collections.unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup)); + final List toDelete = unmodifiableList(new ArrayList<>(snapshotIdsRequiringCleanup)); ensureBelowConcurrencyLimit(repoName, toDelete.get(0).getName(), snapshots, deletionsInProgress); newDelete = new SnapshotDeletionsInProgress.Entry( toDelete, @@ -3353,7 +3359,7 @@ private static void completeListenersIgnoringException(@Nullable List shards( + private static Map shards( SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, Metadata metadata, @@ -3362,7 +3368,7 @@ private static Map shards( RepositoryData repositoryData, String repoName ) { - final Map builder = new HashMap<>(); + final Map builder = new HashMap<>(); final ShardGenerations shardGenerations = repositoryData.shardGenerations(); final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo( repoName, @@ -3396,7 +3402,7 @@ private static Map shards( } final ShardSnapshotStatus shardSnapshotStatus; if (indexRoutingTable == null) { - shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus( + shardSnapshotStatus = new ShardSnapshotStatus( null, ShardState.MISSING, "missing routing table", From b06ddb617c09f65a7804c483d3e3dcc7159c4a29 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 4 Oct 2024 12:48:14 -0400 Subject: [PATCH 188/357] Fix warnings from SLF4J on startup when repository-s3 is installed (#16194) * Fix warnings from SLF4J on startup when repository-s3 is installed Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Fix precommit Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + plugins/repository-s3/build.gradle | 7 ++++--- .../licenses/log4j-slf4j-impl-2.21.0.jar.sha1 | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index ca23cd59e06d5..1a289a8c62000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) +- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) ### Security diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 00decbe4fa9cd..22aa151c92003 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -82,6 +82,8 @@ dependencies { api "joda-time:joda-time:${versions.joda}" api "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" + // network stack api "io.netty:netty-buffer:${versions.netty}" api "io.netty:netty-codec:${versions.netty}" @@ -111,6 +113,7 @@ tasks.named("dependencyLicenses").configure { mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' mapping from: /netty-.*/, to: 'netty' + mapping from: /log4j-.*/, to: 'log4j' } bundlePlugin { @@ -510,9 +513,7 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', + 'org.slf4j.ext.EventData', 'reactor.blockhound.BlockHound$Builder', 'reactor.blockhound.integration.BlockHoundIntegration', diff --git a/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 new file mode 100644 index 0000000000000..0e22f98daa61c --- /dev/null +++ b/plugins/repository-s3/licenses/log4j-slf4j-impl-2.21.0.jar.sha1 @@ -0,0 +1 @@ +911fdb5b1a1df36719c579ecc6f2957b88bce1ab \ No newline at end of file From 421a1cc2270ef02f02fc7621895f8b15ec7282f1 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 4 Oct 2024 22:24:36 +0530 Subject: [PATCH 189/357] Fix unknown parameter source_remote_translog_repository bug (#16192) Signed-off-by: Sachin Kale --- .../RestoreShallowSnapshotV2IT.java | 83 +++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 6 ++ .../RestoreSnapshotRequestBuilder.java | 8 ++ 3 files changed, 97 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index c5a55f16cab2b..24f1141ddbede 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -10,6 +10,8 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; @@ -19,6 +21,7 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobPath; @@ -64,6 +67,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -106,6 +110,18 @@ protected Settings.Builder getRepositorySettings(Path location, boolean shallowC return settingsBuilder; } + protected Settings.Builder getRepositorySettings(String sourceRepository, boolean readOnly) throws ExecutionException, + InterruptedException { + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { sourceRepository }); + GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get(); + RepositoryMetadata rmd = res.repositories().get(0); + return Settings.builder() + .put(rmd.settings()) + .put(BlobStoreRepository.READONLY_SETTING.getKey(), readOnly) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false) + .put(SYSTEM_REPOSITORY_SETTING.getKey(), false); + } + private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { Settings.Builder settingsBuilder = Settings.builder() .put(super.indexSettings()) @@ -802,4 +818,71 @@ public void testInvalidRestoreRequestScenarios() throws Exception { ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); } + + public void testRestoreOperationsUsingDifferentRepos() throws Exception { + disableRepoConsistencyCheck("Remote store repo"); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); + String primary = internalCluster().startDataOnlyNode(); + String indexName1 = "testindex1"; + String snapshotRepoName = "test-snapshot-repo"; + String snapshotName1 = "test-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + // Create repo + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + // Create index + Client client = client(); + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + ensureGreen(indexName1); + + // Index 5 documents, refresh, index 5 documents + final int numDocsInIndex1 = 5; + indexDocuments(client, indexName1, 0, numDocsInIndex1); + refresh(indexName1); + indexDocuments(client, indexName1, numDocsInIndex1, 2 * numDocsInIndex1); + + // Take V2 snapshot + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>()); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + // Create new snapshot, segment and translog repositories + String newSnapshotRepo = "backup-snapshot"; + String newSegmentRepo = "backup-segment"; + String newTranslogRepo = "backup-translog"; + createRepository(newSnapshotRepo, "fs", getRepositorySettings(snapshotRepoName, true)); + createRepository(newSegmentRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true)); + createRepository(newTranslogRepo, "fs", getRepositorySettings(BASE_REMOTE_REPO, true)); + + // Delete index + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertFalse(indexExists(indexName1)); + + // Restore using new repos + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(newSnapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndices(indexName1) + .setSourceRemoteStoreRepository(newSegmentRepo) + .setSourceRemoteTranslogRepository(newTranslogRepo) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + + // Verify restored index's stats + ensureYellowAndNoInitializingShards(indexName1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, 2 * numDocsInIndex1); + + // indexing some new docs and validating + indexDocuments(client, indexName1, 2 * numDocsInIndex1, 3 * numDocsInIndex1); + ensureGreen(indexName1); + assertDocsPresentInIndex(client, indexName1, 3 * numDocsInIndex1); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 469a31407dc5d..f3110cc8f20a5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -652,6 +652,12 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed source_remote_store_repository"); } + } else if (name.equals("source_remote_translog_repository")) { + if (entry.getValue() instanceof String) { + setSourceRemoteTranslogRepository((String) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed source_remote_translog_repository"); + } } else { if (IndicesOptions.isIndicesOptions(name) == false) { throw new IllegalArgumentException("Unknown parameter " + name); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 39eaadf3c8de6..53c9557a621b7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -266,4 +266,12 @@ public RestoreSnapshotRequestBuilder setSourceRemoteStoreRepository(String repos request.setSourceRemoteStoreRepository(repositoryName); return this; } + + /** + * Sets the source remote translog repository name + */ + public RestoreSnapshotRequestBuilder setSourceRemoteTranslogRepository(String repositoryName) { + request.setSourceRemoteTranslogRepository(repositoryName); + return this; + } } From 7a58f5e32fc4ad8c48cb401c4b516fb4cd09856f Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:53:48 +0530 Subject: [PATCH 190/357] Mute snapshot v2 flaky tests (#16193) Signed-off-by: Gaurav Bafna --- .../java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index 97d63d0e2380a..4ecbe221550e6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -479,6 +479,7 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception { assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191") public void testDeleteWhileV2CreateOngoing() throws Exception { final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); @@ -537,6 +538,7 @@ public void testDeleteWhileV2CreateOngoing() throws Exception { assertThat(snapInfo, contains(csr.getSnapshotInfo())); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191") public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); From a81b868facc186728b3d35fd65a6d95621f1dd82 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:55:52 +0530 Subject: [PATCH 191/357] Implementing pagination for _cat/shards (#14641) * Adding _list/shards API Signed-off-by: Harsh Garg --- CHANGELOG.md | 1 + .../shards/TransportCatShardsActionIT.java | 5 +- .../org/opensearch/action/ActionModule.java | 2 + .../cluster/shards/CatShardsRequest.java | 39 +- .../cluster/shards/CatShardsResponse.java | 55 +- .../shards/TransportCatShardsAction.java | 37 +- .../indices/stats/IndicesStatsResponse.java | 2 +- .../pagination/IndexPaginationStrategy.java | 67 ++- .../action/pagination/PageParams.java | 82 +++ .../action/pagination/PageToken.java | 74 +++ .../pagination/PaginationStrategy.java | 10 +- .../pagination/ShardPaginationStrategy.java | 275 +++++++++ .../pagination/package-info.java | 2 +- .../java/org/opensearch/common/Table.java | 2 +- .../java/org/opensearch/rest/RestRequest.java | 8 +- .../rest/action/cat/RestIndicesAction.java | 4 +- .../rest/action/cat/RestShardsAction.java | 53 +- .../opensearch/rest/action/cat/RestTable.java | 2 +- .../rest/action/list/AbstractListAction.java | 6 +- .../action/list/RestIndicesListAction.java | 5 +- .../action/list/RestShardsListAction.java | 74 +++ .../rest/pagination/PageParams.java | 48 -- .../opensearch/rest/pagination/PageToken.java | 42 -- .../cluster/shards/CatShardsRequestTests.java | 112 ++++ .../shards/CatShardsResponseTests.java | 125 ++++- .../IndexPaginationStrategyTests.java | 6 +- .../action/pagination/PageParamsTests.java | 37 ++ .../action/pagination/PageTokenTests.java | 38 ++ .../ShardPaginationStrategyTests.java | 527 ++++++++++++++++++ .../action/cat/RestIndicesActionTests.java | 2 +- .../action/cat/RestShardsActionTests.java | 48 +- .../rest/action/cat/RestTableTests.java | 2 +- .../list/RestShardsListActionTests.java | 76 +++ .../opensearch/test/rest/FakeRestRequest.java | 9 + 34 files changed, 1691 insertions(+), 186 deletions(-) rename server/src/main/java/org/opensearch/{rest => action}/pagination/IndexPaginationStrategy.java (75%) create mode 100644 server/src/main/java/org/opensearch/action/pagination/PageParams.java create mode 100644 server/src/main/java/org/opensearch/action/pagination/PageToken.java rename server/src/main/java/org/opensearch/{rest => action}/pagination/PaginationStrategy.java (85%) create mode 100644 server/src/main/java/org/opensearch/action/pagination/ShardPaginationStrategy.java rename server/src/main/java/org/opensearch/{rest => action}/pagination/package-info.java (86%) create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestShardsListAction.java delete mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageParams.java delete mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageToken.java create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java rename server/src/test/java/org/opensearch/{rest => action}/pagination/IndexPaginationStrategyTests.java (99%) create mode 100644 server/src/test/java/org/opensearch/action/pagination/PageParamsTests.java create mode 100644 server/src/test/java/org/opensearch/action/pagination/PageTokenTests.java create mode 100644 server/src/test/java/org/opensearch/action/pagination/ShardPaginationStrategyTests.java create mode 100644 server/src/test/java/org/opensearch/rest/action/list/RestShardsListActionTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a289a8c62000..cbf19fb0d35be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) +- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java index b86521dedf739..32d5b3db85629 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java @@ -8,7 +8,6 @@ package org.opensearch.action.admin.cluster.shards; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.ShardRouting; @@ -51,9 +50,9 @@ public void testCatShardsWithSuccessResponse() throws InterruptedException { client().execute(CatShardsAction.INSTANCE, shardsRequest, new ActionListener() { @Override public void onResponse(CatShardsResponse catShardsResponse) { - ClusterStateResponse clusterStateResponse = catShardsResponse.getClusterStateResponse(); + List shardRoutings = catShardsResponse.getResponseShards(); IndicesStatsResponse indicesStatsResponse = catShardsResponse.getIndicesStatsResponse(); - for (ShardRouting shard : clusterStateResponse.getState().routingTable().allShards()) { + for (ShardRouting shard : shardRoutings) { assertEquals("test", shard.getIndexName()); assertNotNull(indicesStatsResponse.asMap().get(shard)); } diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 98bcd6ba9c3f8..d6b4a93b34217 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -465,6 +465,7 @@ import org.opensearch.rest.action.list.AbstractListAction; import org.opensearch.rest.action.list.RestIndicesListAction; import org.opensearch.rest.action.list.RestListAction; +import org.opensearch.rest.action.list.RestShardsListAction; import org.opensearch.rest.action.search.RestClearScrollAction; import org.opensearch.rest.action.search.RestCountAction; import org.opensearch.rest.action.search.RestCreatePitAction; @@ -993,6 +994,7 @@ public void initRestHandlers(Supplier nodesInCluster) { // LIST API registerHandler.accept(new RestIndicesListAction(responseLimitSettings)); + registerHandler.accept(new RestShardsListAction()); // Point in time API registerHandler.accept(new RestCreatePitAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java index 76aa25b9a96b5..0319fa103138f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java @@ -8,10 +8,13 @@ package org.opensearch.action.admin.cluster.shards; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.pagination.PageParams; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.tasks.TaskId; import org.opensearch.rest.action.admin.cluster.ClusterAdminTask; @@ -27,13 +30,39 @@ public class CatShardsRequest extends ClusterManagerNodeReadRequest responseShards = new ArrayList<>(); + private PageToken pageToken; public CatShardsResponse() {} public CatShardsResponse(StreamInput in) throws IOException { super(in); + indicesStatsResponse = new IndicesStatsResponse(in); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + nodes = DiscoveryNodes.readFrom(in, null); + responseShards = in.readList(ShardRouting::new); + if (in.readBoolean()) { + pageToken = new PageToken(in); + } + } } @Override public void writeTo(StreamOutput out) throws IOException { - clusterStateResponse.writeTo(out); indicesStatsResponse.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + nodes.writeToWithAttribute(out); + out.writeList(responseShards); + out.writeBoolean(pageToken != null); + if (pageToken != null) { + pageToken.writeTo(out); + } + } } - public void setClusterStateResponse(ClusterStateResponse clusterStateResponse) { - this.clusterStateResponse = clusterStateResponse; + public void setNodes(DiscoveryNodes nodes) { + this.nodes = nodes; } - public ClusterStateResponse getClusterStateResponse() { - return this.clusterStateResponse; + public DiscoveryNodes getNodes() { + return this.nodes; } public void setIndicesStatsResponse(IndicesStatsResponse indicesStatsResponse) { @@ -54,4 +75,20 @@ public void setIndicesStatsResponse(IndicesStatsResponse indicesStatsResponse) { public IndicesStatsResponse getIndicesStatsResponse() { return this.indicesStatsResponse; } + + public void setResponseShards(List responseShards) { + this.responseShards = responseShards; + } + + public List getResponseShards() { + return this.responseShards; + } + + public void setPageToken(PageToken pageToken) { + this.pageToken = pageToken; + } + + public PageToken getPageToken() { + return this.pageToken; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java index 6b073a16f7d28..3dc8c38152a16 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java @@ -12,6 +12,8 @@ import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.pagination.PageParams; +import org.opensearch.action.pagination.ShardPaginationStrategy; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TimeoutTaskCancellationUtility; @@ -57,7 +59,11 @@ public void doExecute(Task parentTask, CatShardsRequest shardsRequest, ActionLis clusterStateRequest.setShouldCancelOnTimeout(true); clusterStateRequest.local(shardsRequest.local()); clusterStateRequest.clusterManagerNodeTimeout(shardsRequest.clusterManagerNodeTimeout()); - clusterStateRequest.clear().nodes(true).routingTable(true).indices(shardsRequest.getIndices()); + if (Objects.isNull(shardsRequest.getPageParams())) { + clusterStateRequest.clear().nodes(true).routingTable(true).indices(shardsRequest.getIndices()); + } else { + clusterStateRequest.clear().nodes(true).routingTable(true).indices(shardsRequest.getIndices()).metadata(true); + } assert parentTask instanceof CancellableTask; clusterStateRequest.setParentTask(client.getLocalNodeId(), parentTask.getId()); @@ -87,13 +93,26 @@ protected void innerOnFailure(Exception e) { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { validateRequestLimit(shardsRequest, clusterStateResponse, cancellableListener); - catShardsResponse.setClusterStateResponse(clusterStateResponse); - IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); - indicesStatsRequest.setShouldCancelOnTimeout(true); - indicesStatsRequest.all(); - indicesStatsRequest.indices(shardsRequest.getIndices()); - indicesStatsRequest.setParentTask(client.getLocalNodeId(), parentTask.getId()); try { + ShardPaginationStrategy paginationStrategy = getPaginationStrategy( + shardsRequest.getPageParams(), + clusterStateResponse + ); + String[] indices = Objects.isNull(paginationStrategy) + ? shardsRequest.getIndices() + : paginationStrategy.getRequestedIndices().toArray(new String[0]); + catShardsResponse.setNodes(clusterStateResponse.getState().getNodes()); + catShardsResponse.setResponseShards( + Objects.isNull(paginationStrategy) + ? clusterStateResponse.getState().routingTable().allShards() + : paginationStrategy.getRequestedEntities() + ); + catShardsResponse.setPageToken(Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken()); + IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); + indicesStatsRequest.setShouldCancelOnTimeout(true); + indicesStatsRequest.all(); + indicesStatsRequest.indices(indices); + indicesStatsRequest.setParentTask(client.getLocalNodeId(), parentTask.getId()); client.admin().indices().stats(indicesStatsRequest, new ActionListener() { @Override public void onResponse(IndicesStatsResponse indicesStatsResponse) { @@ -122,6 +141,10 @@ public void onFailure(Exception e) { } + private ShardPaginationStrategy getPaginationStrategy(PageParams pageParams, ClusterStateResponse clusterStateResponse) { + return Objects.isNull(pageParams) ? null : new ShardPaginationStrategy(pageParams, clusterStateResponse.getState()); + } + private void validateRequestLimit( final CatShardsRequest shardsRequest, final ClusterStateResponse clusterStateResponse, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java index 900a886481fe6..ae989573b39ea 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -64,7 +64,7 @@ public class IndicesStatsResponse extends BroadcastResponse { private Map shardStatsMap; - IndicesStatsResponse(StreamInput in) throws IOException { + public IndicesStatsResponse(StreamInput in) throws IOException { super(in); shards = in.readArray(ShardStats::new, (size) -> new ShardStats[size]); } diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/action/pagination/IndexPaginationStrategy.java similarity index 75% rename from server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java rename to server/src/main/java/org/opensearch/action/pagination/IndexPaginationStrategy.java index f89ab14e4b24d..6180d098a851d 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/action/pagination/IndexPaginationStrategy.java @@ -6,12 +6,11 @@ * compatible open source license. */ -package org.opensearch.rest.pagination; +package org.opensearch.action.pagination; import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.Nullable; import java.util.ArrayList; import java.util.Comparator; @@ -20,8 +19,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; - /** * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. * The strategy considers create timestamps of indices as the keys to iterate over pages. @@ -31,13 +28,13 @@ public class IndexPaginationStrategy implements PaginationStrategy { private static final String DEFAULT_INDICES_PAGINATED_ENTITY = "indices"; - private static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { + protected static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { if (metadata1.getCreationDate() == metadata2.getCreationDate()) { return metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); } return Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); }; - private static final Comparator DESC_COMPARATOR = (metadata1, metadata2) -> { + protected static final Comparator DESC_COMPARATOR = (metadata1, metadata2) -> { if (metadata1.getCreationDate() == metadata2.getCreationDate()) { return metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); } @@ -48,12 +45,18 @@ public class IndexPaginationStrategy implements PaginationStrategy { private final List requestedIndices; public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) { + + IndexStrategyToken requestedToken = Objects.isNull(pageParams.getRequestedToken()) || pageParams.getRequestedToken().isEmpty() + ? null + : new IndexStrategyToken(pageParams.getRequestedToken()); // Get list of indices metadata sorted by their creation time and filtered by the last sent index - List sortedIndices = PaginationStrategy.getSortedIndexMetadata( + List sortedIndices = getEligibleIndices( clusterState, - getMetadataFilter(pageParams.getRequestedToken(), pageParams.getSort()), - PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) ? ASC_COMPARATOR : DESC_COMPARATOR + pageParams.getSort(), + Objects.isNull(requestedToken) ? null : requestedToken.lastIndexName, + Objects.isNull(requestedToken) ? null : requestedToken.lastIndexCreationTime ); + // Trim sortedIndicesList to get the list of indices metadata to be sent as response List metadataSublist = getMetadataSubList(sortedIndices, pageParams.getSize()); // Get list of index names from the trimmed metadataSublist @@ -65,25 +68,44 @@ public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) ); } - private static Predicate getMetadataFilter(String requestedTokenStr, String sortOrder) { - boolean isAscendingSort = sortOrder.equals(PARAM_ASC_SORT_VALUE); - IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() - ? null - : new IndexStrategyToken(requestedTokenStr); - if (Objects.isNull(requestedToken)) { + private static List getEligibleIndices( + ClusterState clusterState, + String sortOrder, + String lastIndexName, + Long lastIndexCreationTime + ) { + if (Objects.isNull(lastIndexName) || Objects.isNull(lastIndexCreationTime)) { + return PaginationStrategy.getSortedIndexMetadata( + clusterState, + PageParams.PARAM_ASC_SORT_VALUE.equals(sortOrder) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + } else { + return PaginationStrategy.getSortedIndexMetadata( + clusterState, + getMetadataFilter(sortOrder, lastIndexName, lastIndexCreationTime), + PageParams.PARAM_ASC_SORT_VALUE.equals(sortOrder) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + } + } + + private static Predicate getMetadataFilter(String sortOrder, String lastIndexName, Long lastIndexCreationTime) { + if (Objects.isNull(lastIndexName) || Objects.isNull(lastIndexCreationTime)) { return indexMetadata -> true; } + return getIndexCreateTimeFilter(sortOrder, lastIndexName, lastIndexCreationTime); + } + + protected static Predicate getIndexCreateTimeFilter(String sortOrder, String lastIndexName, Long lastIndexCreationTime) { + boolean isAscendingSort = sortOrder.equals(PageParams.PARAM_ASC_SORT_VALUE); return metadata -> { - if (metadata.getIndex().getName().equals(requestedToken.lastIndexName)) { - return false; - } else if (metadata.getCreationDate() == requestedToken.lastIndexCreationTime) { + if (metadata.getCreationDate() == lastIndexCreationTime) { return isAscendingSort - ? metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) > 0 - : metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) < 0; + ? metadata.getIndex().getName().compareTo(lastIndexName) > 0 + : metadata.getIndex().getName().compareTo(lastIndexName) < 0; } return isAscendingSort - ? metadata.getCreationDate() > requestedToken.lastIndexCreationTime - : metadata.getCreationDate() < requestedToken.lastIndexCreationTime; + ? metadata.getCreationDate() > lastIndexCreationTime + : metadata.getCreationDate() < lastIndexCreationTime; }; } @@ -105,7 +127,6 @@ private PageToken getResponseToken(final int pageSize, final int totalIndices, I } @Override - @Nullable public PageToken getResponseToken() { return pageToken; } diff --git a/server/src/main/java/org/opensearch/action/pagination/PageParams.java b/server/src/main/java/org/opensearch/action/pagination/PageParams.java new file mode 100644 index 0000000000000..03de1aa465a15 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/pagination/PageParams.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * + * Class specific to paginated queries, which will contain common query params required by a paginated API. + */ +@PublicApi(since = "3.0.0") +public class PageParams implements Writeable { + + public static final String PARAM_SORT = "sort"; + public static final String PARAM_NEXT_TOKEN = "next_token"; + public static final String PARAM_SIZE = "size"; + public static final String PARAM_ASC_SORT_VALUE = "asc"; + public static final String PARAM_DESC_SORT_VALUE = "desc"; + + private final String requestedTokenStr; + private final String sort; + private final int size; + + public PageParams(StreamInput in) throws IOException { + this.requestedTokenStr = in.readOptionalString(); + this.sort = in.readOptionalString(); + this.size = in.readInt(); + } + + public PageParams(String requestedToken, String sort, int size) { + this.requestedTokenStr = requestedToken; + this.sort = sort; + this.size = size; + } + + public String getSort() { + return sort; + } + + public String getRequestedToken() { + return requestedTokenStr; + } + + public int getSize() { + return size; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(requestedTokenStr); + out.writeOptionalString(sort); + out.writeInt(size); + } + + // Overriding equals and hashcode for tests + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PageParams that = (PageParams) o; + return this.size == that.size + && Objects.equals(this.requestedTokenStr, that.requestedTokenStr) + && Objects.equals(this.sort, that.sort); + } + + @Override + public int hashCode() { + return Objects.hash(requestedTokenStr, sort, size); + } +} diff --git a/server/src/main/java/org/opensearch/action/pagination/PageToken.java b/server/src/main/java/org/opensearch/action/pagination/PageToken.java new file mode 100644 index 0000000000000..44d4079681501 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/pagination/PageToken.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * Pagination response metadata for a paginated query. + * @opensearch.internal + */ +public class PageToken implements Writeable { + + public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; + + /** + * String denoting the next_token of paginated response, which will be used to fetch next page (if any). + */ + private final String nextToken; + + /** + * String denoting the element which is being paginated (for e.g. shards, indices..). + */ + private final String paginatedEntity; + + public PageToken(String nextToken, String paginatedElement) { + assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; + this.nextToken = nextToken; + this.paginatedEntity = paginatedElement; + } + + public PageToken(StreamInput in) throws IOException { + this.nextToken = in.readOptionalString(); + this.paginatedEntity = in.readString(); + } + + public String getNextToken() { + return nextToken; + } + + public String getPaginatedEntity() { + return paginatedEntity; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(nextToken); + out.writeString(paginatedEntity); + } + + // Overriding equals and hashcode for testing + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PageToken that = (PageToken) o; + return Objects.equals(this.nextToken, that.nextToken) && Objects.equals(this.paginatedEntity, that.paginatedEntity); + } + + @Override + public int hashCode() { + return Objects.hash(nextToken, paginatedEntity); + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java similarity index 85% rename from server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java rename to server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java index 7f9825a7cc09b..edb2489b1e4f0 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.rest.pagination; +package org.opensearch.action.pagination; import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; @@ -55,6 +55,14 @@ static List getSortedIndexMetadata( return clusterState.metadata().indices().values().stream().filter(filterPredicate).sorted(comparator).collect(Collectors.toList()); } + /** + * + * Utility method to get list of indices sorted as per {@param comparator}. + */ + static List getSortedIndexMetadata(final ClusterState clusterState, Comparator comparator) { + return clusterState.metadata().indices().values().stream().sorted(comparator).collect(Collectors.toList()); + } + static String encryptStringToken(String tokenString) { if (Objects.isNull(tokenString)) { return null; diff --git a/server/src/main/java/org/opensearch/action/pagination/ShardPaginationStrategy.java b/server/src/main/java/org/opensearch/action/pagination/ShardPaginationStrategy.java new file mode 100644 index 0000000000000..1eb364c883e60 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/pagination/ShardPaginationStrategy.java @@ -0,0 +1,275 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.ShardRouting; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; + +import static org.opensearch.action.pagination.IndexPaginationStrategy.ASC_COMPARATOR; +import static org.opensearch.action.pagination.IndexPaginationStrategy.DESC_COMPARATOR; + +/** + * This strategy can be used by the Rest APIs wanting to paginate the responses based on Shards. + * The strategy considers create timestamps of indices and shardID as the keys to iterate over pages. + * + * @opensearch.internal + */ +public class ShardPaginationStrategy implements PaginationStrategy { + + private static final String DEFAULT_SHARDS_PAGINATED_ENTITY = "shards"; + + private PageData pageData; + + public ShardPaginationStrategy(PageParams pageParams, ClusterState clusterState) { + ShardStrategyToken shardStrategyToken = getShardStrategyToken(pageParams.getRequestedToken()); + // Get list of indices metadata sorted by their creation time and filtered by the last sent index + List filteredIndices = getEligibleIndices( + clusterState, + pageParams.getSort(), + Objects.isNull(shardStrategyToken) ? null : shardStrategyToken.lastIndexName, + Objects.isNull(shardStrategyToken) ? null : shardStrategyToken.lastIndexCreationTime + ); + // Get the list of shards and indices belonging to current page. + this.pageData = getPageData( + filteredIndices, + clusterState.getRoutingTable().getIndicesRouting(), + shardStrategyToken, + pageParams.getSize() + ); + } + + private static List getEligibleIndices( + ClusterState clusterState, + String sortOrder, + String lastIndexName, + Long lastIndexCreationTime + ) { + if (Objects.isNull(lastIndexName) || Objects.isNull(lastIndexCreationTime)) { + return PaginationStrategy.getSortedIndexMetadata( + clusterState, + PageParams.PARAM_ASC_SORT_VALUE.equals(sortOrder) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + } else { + return PaginationStrategy.getSortedIndexMetadata( + clusterState, + getMetadataFilter(sortOrder, lastIndexName, lastIndexCreationTime), + PageParams.PARAM_ASC_SORT_VALUE.equals(sortOrder) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + } + } + + private static Predicate getMetadataFilter(String sortOrder, String lastIndexName, Long lastIndexCreationTime) { + if (Objects.isNull(lastIndexName) || Objects.isNull(lastIndexCreationTime)) { + return indexMetadata -> true; + } + return indexNameFilter(lastIndexName).or( + IndexPaginationStrategy.getIndexCreateTimeFilter(sortOrder, lastIndexName, lastIndexCreationTime) + ); + } + + private static Predicate indexNameFilter(String lastIndexName) { + return metadata -> metadata.getIndex().getName().equals(lastIndexName); + } + + /** + * Will be used to get the list of shards and respective indices to which they belong, + * which are to be displayed in a page. + * Note: All shards for a shardID will always be present in the same page. + */ + private PageData getPageData( + List filteredIndices, + Map indicesRouting, + final ShardStrategyToken token, + final int numShardsRequired + ) { + List shardRoutings = new ArrayList<>(); + List indices = new ArrayList<>(); + int shardCount = 0; + IndexMetadata lastAddedIndex = null; + + // iterate over indices until shardCount is less than numShardsRequired + for (IndexMetadata indexMetadata : filteredIndices) { + String indexName = indexMetadata.getIndex().getName(); + boolean indexShardsAdded = false; + // Always start from shardID 0 for all indices except for the first one which might be same as the last sent + // index. To identify if an index is same as last sent index, verify both the index name and creaton time. + int startShardId = shardCount == 0 ? getStartShardIdForPageIndex(token, indexName, indexMetadata.getCreationDate()) : 0; + Map indexShardRoutingTable = indicesRouting.get(indexName).getShards(); + for (; startShardId < indexShardRoutingTable.size(); startShardId++) { + if (indexShardRoutingTable.get(startShardId).size() > numShardsRequired) { + throw new IllegalArgumentException( + "size value should be greater than the replica count of all indices, so that all primary and replicas of a shard show up in single page" + ); + } + shardCount += indexShardRoutingTable.get(startShardId).size(); + if (shardCount > numShardsRequired) { + break; + } + shardRoutings.addAll(indexShardRoutingTable.get(startShardId).shards()); + indexShardsAdded = true; + } + + if (indexShardsAdded) { + lastAddedIndex = indexMetadata; + indices.add(indexName); + } + + if (shardCount > numShardsRequired) { + return new PageData( + shardRoutings, + indices, + new PageToken( + new ShardStrategyToken( + lastAddedIndex.getIndex().getName(), + shardRoutings.get(shardRoutings.size() - 1).id(), + lastAddedIndex.getCreationDate() + ).generateEncryptedToken(), + DEFAULT_SHARDS_PAGINATED_ENTITY + ) + ); + } + } + + // If all the shards of filtered indices list have been included in pageShardRoutings, then no more + // shards are remaining to be fetched, and the next_token should thus be null. + return new PageData(shardRoutings, indices, new PageToken(null, DEFAULT_SHARDS_PAGINATED_ENTITY)); + } + + private ShardStrategyToken getShardStrategyToken(String requestedToken) { + return Objects.isNull(requestedToken) || requestedToken.isEmpty() ? null : new ShardStrategyToken(requestedToken); + } + + /** + * Provides the shardId to start an index which is to be included in the page. If the index is same as + * lastIndex, start from the shardId next to lastShardId, else always start from 0. + */ + private int getStartShardIdForPageIndex(ShardStrategyToken token, final String pageIndexName, final long pageIndexCreationTime) { + return Objects.isNull(token) ? 0 + : token.lastIndexName.equals(pageIndexName) && token.lastIndexCreationTime == pageIndexCreationTime ? token.lastShardId + 1 + : 0; + } + + @Override + public PageToken getResponseToken() { + return pageData.pageToken; + } + + @Override + public List getRequestedEntities() { + return pageData.shardRoutings; + } + + public List getRequestedIndices() { + return pageData.indices; + } + + /** + * TokenParser to be used by {@link ShardPaginationStrategy}. + * TToken would look like: LastShardIdOfPage + | + CreationTimeOfLastRespondedIndex + | + NameOfLastRespondedIndex + */ + public static class ShardStrategyToken { + + private static final String JOIN_DELIMITER = "|"; + private static final String SPLIT_REGEX = "\\|"; + private static final int SHARD_ID_POS_IN_TOKEN = 0; + private static final int CREATE_TIME_POS_IN_TOKEN = 1; + private static final int INDEX_NAME_POS_IN_TOKEN = 2; + + /** + * Denotes the shardId of the last shard in the response. + * Will be used to identify the next shard to start the page from, in case the shards of an index + * get split across pages. + */ + private final int lastShardId; + /** + * Represents creation times of last index which was displayed in the page. + * Used to identify the new start point in case the indices get created/deleted while queries are executed. + */ + private final long lastIndexCreationTime; + + /** + * Represents name of the last index which was displayed in the page. + * Used to identify whether the sorted list of indices has changed or not. + */ + private final String lastIndexName; + + public ShardStrategyToken(String requestedTokenString) { + validateShardStrategyToken(requestedTokenString); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + this.lastShardId = Integer.parseInt(decryptedTokenElements[SHARD_ID_POS_IN_TOKEN]); + this.lastIndexCreationTime = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + this.lastIndexName = decryptedTokenElements[INDEX_NAME_POS_IN_TOKEN]; + } + + public ShardStrategyToken(String lastIndexName, int lastShardId, long lastIndexCreationTime) { + Objects.requireNonNull(lastIndexName, "index name should be provided"); + this.lastIndexName = lastIndexName; + this.lastShardId = lastShardId; + this.lastIndexCreationTime = lastIndexCreationTime; + } + + public String generateEncryptedToken() { + return PaginationStrategy.encryptStringToken( + String.join(JOIN_DELIMITER, String.valueOf(lastShardId), String.valueOf(lastIndexCreationTime), lastIndexName) + ); + } + + /** + * Will perform simple validations on token received in the request. + * Token should be base64 encoded, and should contain the expected number of elements separated by "|". + * Timestamps should also be a valid long. + * + * @param requestedTokenStr string denoting the encoded token requested by the user. + */ + public static void validateShardStrategyToken(String requestedTokenStr) { + Objects.requireNonNull(requestedTokenStr, "requestedTokenString can not be null"); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenStr); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + if (decryptedTokenElements.length != 3) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + try { + int shardId = Integer.parseInt(decryptedTokenElements[SHARD_ID_POS_IN_TOKEN]); + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + if (shardId < 0 || creationTimeOfLastRespondedIndex <= 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + } + + /** + * Private utility class to consolidate the page details of shard strategy. Will be used to set all the details at once, + * to avoid parsing collections multiple times. + */ + private static class PageData { + private final List shardRoutings; + private final List indices; + private final PageToken pageToken; + + PageData(List shardRoutings, List indices, PageToken pageToken) { + this.shardRoutings = shardRoutings; + this.indices = indices; + this.pageToken = pageToken; + } + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/package-info.java b/server/src/main/java/org/opensearch/action/pagination/package-info.java similarity index 86% rename from server/src/main/java/org/opensearch/rest/pagination/package-info.java rename to server/src/main/java/org/opensearch/action/pagination/package-info.java index 324b8a6c46f88..4cf4c7d05adaf 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/package-info.java +++ b/server/src/main/java/org/opensearch/action/pagination/package-info.java @@ -9,4 +9,4 @@ /** * Exposes utilities for Rest actions to paginate responses. */ -package org.opensearch.rest.pagination; +package org.opensearch.action.pagination; diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index 133ec3052e6c9..c2dd770da845b 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -32,9 +32,9 @@ package org.opensearch.common; +import org.opensearch.action.pagination.PageToken; import org.opensearch.common.time.DateFormatter; import org.opensearch.core.common.Strings; -import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index f241b567c3204..4d5bad2e7fd49 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -33,6 +33,7 @@ package org.opensearch.rest; import org.opensearch.OpenSearchParseException; +import org.opensearch.action.pagination.PageParams; import org.opensearch.common.Booleans; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Nullable; @@ -51,7 +52,6 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; -import org.opensearch.rest.pagination.PageParams; import java.io.IOException; import java.io.InputStream; @@ -66,11 +66,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.opensearch.action.pagination.PageParams.PARAM_NEXT_TOKEN; +import static org.opensearch.action.pagination.PageParams.PARAM_SIZE; +import static org.opensearch.action.pagination.PageParams.PARAM_SORT; import static org.opensearch.common.unit.TimeValue.parseTimeValue; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; -import static org.opensearch.rest.pagination.PageParams.PARAM_NEXT_TOKEN; -import static org.opensearch.rest.pagination.PageParams.PARAM_SIZE; -import static org.opensearch.rest.pagination.PageParams.PARAM_SORT; /** * REST Request diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 7562e09ddd704..b566ba9bbb8e4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -43,6 +43,8 @@ import org.opensearch.action.admin.indices.stats.IndexStats; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; +import org.opensearch.action.pagination.IndexPaginationStrategy; +import org.opensearch.action.pagination.PageToken; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; @@ -65,8 +67,6 @@ import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; import org.opensearch.rest.action.list.AbstractListAction; -import org.opensearch.rest.pagination.IndexPaginationStrategy; -import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index c8b4e7472927e..04212b6b057fe 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -35,11 +35,12 @@ import org.opensearch.action.admin.cluster.shards.CatShardsAction; import org.opensearch.action.admin.cluster.shards.CatShardsRequest; import org.opensearch.action.admin.cluster.shards.CatShardsResponse; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.pagination.PageToken; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.Table; @@ -63,6 +64,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.action.list.AbstractListAction; import org.opensearch.search.suggest.completion.CompletionStats; import java.time.Instant; @@ -80,7 +82,7 @@ * * @opensearch.api */ -public class RestShardsAction extends AbstractCatAction { +public class RestShardsAction extends AbstractListAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestShardsAction.class); @@ -119,20 +121,32 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli shardsRequest.setCancelAfterTimeInterval(request.paramAsTime("cancel_after_time_interval", NO_TIMEOUT)); shardsRequest.setIndices(indices); shardsRequest.setRequestLimitCheckSupported(isRequestLimitCheckSupported()); + shardsRequest.setPageParams(pageParams); parseDeprecatedMasterTimeoutParameter(shardsRequest, request, deprecationLogger, getName()); return channel -> client.execute(CatShardsAction.INSTANCE, shardsRequest, new RestResponseListener(channel) { @Override public RestResponse buildResponse(CatShardsResponse catShardsResponse) throws Exception { - ClusterStateResponse clusterStateResponse = catShardsResponse.getClusterStateResponse(); - IndicesStatsResponse indicesStatsResponse = catShardsResponse.getIndicesStatsResponse(); - return RestTable.buildResponse(buildTable(request, clusterStateResponse, indicesStatsResponse), channel); + return RestTable.buildResponse( + buildTable( + request, + catShardsResponse.getNodes(), + catShardsResponse.getIndicesStatsResponse(), + catShardsResponse.getResponseShards(), + catShardsResponse.getPageToken() + ), + channel + ); } }); } @Override protected Table getTableWithHeader(final RestRequest request) { - Table table = new Table(); + return getTableWithHeader(request, null); + } + + protected Table getTableWithHeader(final RestRequest request, final PageToken pageToken) { + Table table = new Table(pageToken); table.startHeaders() .addCell("index", "default:true;alias:i,idx;desc:index name") .addCell("shard", "default:true;alias:s,sh;desc:shard name") @@ -301,10 +315,16 @@ private static Object getOrNull(S stats, Function accessor, Functio } // package private for testing - Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) { - Table table = getTableWithHeader(request); - - for (ShardRouting shard : state.getState().routingTable().allShards()) { + Table buildTable( + RestRequest request, + DiscoveryNodes nodes, + IndicesStatsResponse stats, + List responseShards, + PageToken pageToken + ) { + Table table = getTableWithHeader(request, pageToken); + + for (ShardRouting shard : responseShards) { ShardStats shardStats = stats.asMap().get(shard); CommonStats commonStats = null; CommitStats commitStats = null; @@ -331,13 +351,13 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getCount)); table.addCell(getOrNull(commonStats, CommonStats::getStore, StoreStats::getSize)); if (shard.assignedToNode()) { - String ip = state.getState().nodes().get(shard.currentNodeId()).getHostAddress(); + String ip = nodes.get(shard.currentNodeId()).getHostAddress(); String nodeId = shard.currentNodeId(); StringBuilder name = new StringBuilder(); - name.append(state.getState().nodes().get(shard.currentNodeId()).getName()); + name.append(nodes.get(shard.currentNodeId()).getName()); if (shard.relocating()) { - String reloIp = state.getState().nodes().get(shard.relocatingNodeId()).getHostAddress(); - String reloNme = state.getState().nodes().get(shard.relocatingNodeId()).getName(); + String reloIp = nodes.get(shard.relocatingNodeId()).getHostAddress(); + String reloNme = nodes.get(shard.relocatingNodeId()).getName(); String reloNodeId = shard.relocatingNodeId(); name.append(" -> "); name.append(reloIp); @@ -460,4 +480,9 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe return table; } + + @Override + public boolean isActionPaginated() { + return false; + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index d622dd7a956f4..e27c8212e16a0 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -61,7 +61,7 @@ import java.util.Objects; import java.util.Set; -import static org.opensearch.rest.pagination.PageToken.PAGINATED_RESPONSE_NEXT_TOKEN_KEY; +import static org.opensearch.action.pagination.PageToken.PAGINATED_RESPONSE_NEXT_TOKEN_KEY; /** * a REST table diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java index f3d6d6653a550..5bdee750fd96f 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -8,16 +8,16 @@ package org.opensearch.rest.action.list; +import org.opensearch.action.pagination.PageParams; import org.opensearch.client.node.NodeClient; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.AbstractCatAction; -import org.opensearch.rest.pagination.PageParams; import java.io.IOException; import java.util.Objects; -import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; -import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; +import static org.opensearch.action.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.action.pagination.PageParams.PARAM_DESC_SORT_VALUE; /** * Base Transport action class for _list API. diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index 971cab0535be9..bd6a9fb09aafe 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,13 +9,13 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.pagination.IndexPaginationStrategy; +import org.opensearch.action.pagination.PageParams; import org.opensearch.common.breaker.ResponseLimitSettings; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; -import org.opensearch.rest.pagination.IndexPaginationStrategy; -import org.opensearch.rest.pagination.PageParams; import java.util.Iterator; import java.util.List; @@ -50,7 +50,6 @@ public String getName() { return "list_indices_action"; } - @Override protected void documentation(StringBuilder sb) { sb.append("/_list/indices\n"); sb.append("/_list/indices/{index}\n"); diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestShardsListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestShardsListAction.java new file mode 100644 index 0000000000000..ba11592398174 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestShardsListAction.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.action.pagination.PageParams; +import org.opensearch.action.pagination.ShardPaginationStrategy; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.RestShardsAction; + +import java.util.List; +import java.util.Objects; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * _list API action to output shards in pages. + * + * @opensearch.api + */ +public class RestShardsListAction extends RestShardsAction { + + protected static final int MAX_SUPPORTED_LIST_SHARDS_PAGE_SIZE = 20000; + protected static final int MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE = 2000; + + @Override + public List routes() { + return unmodifiableList(asList(new Route(GET, "/_list/shards"), new Route(GET, "/_list/shards/{index}"))); + } + + @Override + public String getName() { + return "list_shards_action"; + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_list/shards\n"); + sb.append("/_list/shards/{index}\n"); + } + + @Override + public boolean isActionPaginated() { + return true; + } + + @Override + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = super.validateAndGetPageParams(restRequest); + // validate max supported pageSize + if (pageParams.getSize() < MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE) { + throw new IllegalArgumentException("size should at least be [" + MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE + "]"); + } else if (pageParams.getSize() > MAX_SUPPORTED_LIST_SHARDS_PAGE_SIZE) { + throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_SHARDS_PAGE_SIZE + "]"); + } + // Next Token in the request will be validated by the ShardStrategyToken itself. + if (Objects.nonNull(pageParams.getRequestedToken())) { + ShardPaginationStrategy.ShardStrategyToken.validateShardStrategyToken(pageParams.getRequestedToken()); + } + return pageParams; + } + + @Override + protected int defaultPageSize() { + return MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE; + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageParams.java b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java deleted file mode 100644 index 9b2074bc3fed0..0000000000000 --- a/server/src/main/java/org/opensearch/rest/pagination/PageParams.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.pagination; - -import org.opensearch.common.annotation.PublicApi; - -/** - * - * Class specific to paginated queries, which will contain common query params required by a paginated API. - */ -@PublicApi(since = "3.0.0") -public class PageParams { - - public static final String PARAM_SORT = "sort"; - public static final String PARAM_NEXT_TOKEN = "next_token"; - public static final String PARAM_SIZE = "size"; - public static final String PARAM_ASC_SORT_VALUE = "asc"; - public static final String PARAM_DESC_SORT_VALUE = "desc"; - - private final String requestedTokenStr; - private final String sort; - private final int size; - - public PageParams(String requestedToken, String sort, int size) { - this.requestedTokenStr = requestedToken; - this.sort = sort; - this.size = size; - } - - public String getSort() { - return sort; - } - - public String getRequestedToken() { - return requestedTokenStr; - } - - public int getSize() { - return size; - } - -} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java deleted file mode 100644 index d62e1be695715..0000000000000 --- a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.pagination; - -/** - * Pagination response metadata for a paginated query. - * @opensearch.internal - */ -public class PageToken { - - public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; - - /** - * String denoting the next_token of paginated response, which will be used to fetch next page (if any). - */ - private final String nextToken; - - /** - * String denoting the element which is being paginated (for e.g. shards, indices..). - */ - private final String paginatedEntity; - - public PageToken(String nextToken, String paginatedElement) { - assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; - this.nextToken = nextToken; - this.paginatedEntity = paginatedElement; - } - - public String getNextToken() { - return nextToken; - } - - public String getPaginatedEntity() { - return paginatedEntity; - } -} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java new file mode 100644 index 0000000000000..f4215f54c1e21 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.shards; + +import org.opensearch.Version; +import org.opensearch.action.pagination.PageParams; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; + +public class CatShardsRequestTests extends OpenSearchTestCase { + + public void testSerializationWithDefaultParameters() throws Exception { + CatShardsRequest request = new CatShardsRequest(); + Version version = Version.CURRENT; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + request.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsRequest deserialized = new CatShardsRequest(in); + assertNull(deserialized.getPageParams()); + assertNull(deserialized.getCancelAfterTimeInterval()); + assertEquals(0, deserialized.getIndices().length); + assertFalse(deserialized.isRequestLimitCheckSupported()); + } + } + } + + public void testSerializationWithStringPageParamsNull() throws Exception { + CatShardsRequest catShardsRequest = new CatShardsRequest(); + catShardsRequest.setPageParams(new PageParams(null, null, randomIntBetween(1, 5))); + int numIndices = randomIntBetween(1, 5); + String[] indices = new String[numIndices]; + for (int i = 0; i < numIndices; i++) { + indices[i] = randomAlphaOfLengthBetween(3, 10); + } + catShardsRequest.setIndices(indices); + catShardsRequest.setCancelAfterTimeInterval(TimeValue.timeValueMillis(randomIntBetween(1, 5))); + catShardsRequest.setRequestLimitCheckSupported(true); + + Version version = Version.CURRENT; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + catShardsRequest.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsRequest deserialized = new CatShardsRequest(in); + // asserting pageParams of deserialized request + assertEquals(catShardsRequest.getPageParams(), deserialized.getPageParams()); + // assert indices + assertArrayEquals(catShardsRequest.getIndices(), deserialized.getIndices()); + // assert timeout + assertEquals(catShardsRequest.getCancelAfterTimeInterval(), deserialized.getCancelAfterTimeInterval()); + assertTrue(deserialized.isRequestLimitCheckSupported()); + } + } + } + + public void testSerializationWithPageParamsSet() throws Exception { + CatShardsRequest catShardsRequest = new CatShardsRequest(); + catShardsRequest.setPageParams( + new PageParams(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomIntBetween(1, 5)) + ); + Version version = Version.CURRENT; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + catShardsRequest.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsRequest deserialized = new CatShardsRequest(in); + + // asserting pageParams of deserialized request + assertEquals(catShardsRequest.getPageParams(), deserialized.getPageParams()); + assertEquals(0, deserialized.getIndices().length); + assertNull(deserialized.getCancelAfterTimeInterval()); + assertFalse(deserialized.isRequestLimitCheckSupported()); + } + } + } + + public void testSerializationWithOlderVersionsParametersNotSerialized() throws Exception { + CatShardsRequest catShardsRequest = new CatShardsRequest(); + catShardsRequest.setPageParams( + new PageParams(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10), randomIntBetween(1, 5)) + ); + catShardsRequest.setCancelAfterTimeInterval(TimeValue.timeValueMillis(randomIntBetween(1, 5))); + catShardsRequest.setIndices(new String[2]); + + Version version = VersionUtils.getPreviousVersion(Version.CURRENT); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + catShardsRequest.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsRequest deserialized = new CatShardsRequest(in); + assertNull(deserialized.getPageParams()); + assertNull(deserialized.getIndices()); + assertNull(deserialized.getCancelAfterTimeInterval()); + assertFalse(deserialized.isRequestLimitCheckSupported()); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java index d0b98f4d286ae..11b1d5567d9fb 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shards/CatShardsResponseTests.java @@ -32,24 +32,46 @@ package org.opensearch.action.admin.indices.shards; +import org.opensearch.Version; import org.opensearch.action.admin.cluster.shards.CatShardsResponse; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; -import org.opensearch.cluster.ClusterName; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.pagination.PageToken; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.UUIDs; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.shard.ShardPath; import org.opensearch.test.OpenSearchTestCase; -import static org.junit.Assert.assertEquals; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.mockito.Mockito.mock; public class CatShardsResponseTests extends OpenSearchTestCase { + private static final String TEST_NODE_ID = "test_node_id"; private final CatShardsResponse catShardsResponse = new CatShardsResponse(); - public void testGetAndSetClusterStateResponse() { - ClusterName clusterName = new ClusterName("1"); - ClusterStateResponse clusterStateResponse = new ClusterStateResponse(clusterName, null, false); - catShardsResponse.setClusterStateResponse(clusterStateResponse); - - assertEquals(clusterStateResponse, catShardsResponse.getClusterStateResponse()); + public void testGetAndSetNodes() { + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); + catShardsResponse.setNodes(discoveryNodes); + assertEquals(discoveryNodes, catShardsResponse.getNodes()); } public void testGetAndSetIndicesStatsResponse() { @@ -58,4 +80,89 @@ public void testGetAndSetIndicesStatsResponse() { assertEquals(indicesStatsResponse, catShardsResponse.getIndicesStatsResponse()); } + + public void testSerialization() throws IOException { + CatShardsResponse response = new CatShardsResponse(); + response.setIndicesStatsResponse(getIndicesStatsResponse()); + response.setNodes(getDiscoveryNodes()); + response.setPageToken(new PageToken("token", "test")); + Version version = Version.CURRENT; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + response.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsResponse deserialized = new CatShardsResponse(in); + assertEquals(response.getPageToken(), deserialized.getPageToken()); + assertNotNull(deserialized.getNodes()); + assertEquals(1, deserialized.getNodes().getNodes().size()); + assertNotNull(deserialized.getNodes().getNodes().get(TEST_NODE_ID)); + assertNotNull(deserialized.getIndicesStatsResponse()); + assertEquals( + response.getIndicesStatsResponse().getShards().length, + deserialized.getIndicesStatsResponse().getShards().length + ); + assertTrue(deserialized.getResponseShards().isEmpty()); + } + } + } + + public void testSerializationWithOnlyStatsAndNodesInResponse() throws IOException { + CatShardsResponse response = new CatShardsResponse(); + response.setIndicesStatsResponse(getIndicesStatsResponse()); + response.setNodes(getDiscoveryNodes()); + Version version = Version.CURRENT; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(version); + response.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(version); + CatShardsResponse deserialized = new CatShardsResponse(in); + assertEquals(response.getPageToken(), deserialized.getPageToken()); + assertNotNull(deserialized.getNodes()); + assertEquals(1, deserialized.getNodes().getNodes().size()); + assertNotNull(deserialized.getNodes().getNodes().get(TEST_NODE_ID)); + assertNotNull(deserialized.getIndicesStatsResponse()); + assertEquals( + response.getIndicesStatsResponse().getShards().length, + deserialized.getIndicesStatsResponse().getShards().length + ); + assertTrue(deserialized.getResponseShards().isEmpty()); + } + } + } + + private DiscoveryNodes getDiscoveryNodes() throws UnknownHostException { + DiscoveryNodes.Builder dnBuilder = new DiscoveryNodes.Builder(); + InetAddress inetAddress = InetAddress.getByAddress(new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1 }); + TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); + dnBuilder.add(new DiscoveryNode(TEST_NODE_ID, TEST_NODE_ID, transportAddress, emptyMap(), emptySet(), Version.CURRENT)); + return dnBuilder.build(); + } + + private IndicesStatsResponse getIndicesStatsResponse() { + List shards = new ArrayList<>(); + int noOfIndexes = randomIntBetween(2, 5); + + for (int indCnt = 0; indCnt < noOfIndexes; indCnt++) { + Index index = createIndex(randomAlphaOfLength(9)); + int numShards = randomIntBetween(1, 5); + for (int shardId = 0; shardId < numShards; shardId++) { + ShardId shId = new ShardId(index, shardId); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardId)); + ShardPath shardPath = new ShardPath(false, path, path, shId); + ShardRouting routing = createShardRouting(shId, (shardId == 0)); + shards.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null)); + } + } + return new IndicesStatsResponse(shards.toArray(new ShardStats[0]), 0, 0, 0, null); + } + + private ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { + return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); + } + + private Index createIndex(String indexName) { + return new Index(indexName, UUIDs.base64UUID()); + } } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/action/pagination/IndexPaginationStrategyTests.java similarity index 99% rename from server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java rename to server/src/test/java/org/opensearch/action/pagination/IndexPaginationStrategyTests.java index 01464b489e26e..9900810bf3eb9 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/action/pagination/IndexPaginationStrategyTests.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.rest.pagination; +package org.opensearch.action.pagination; import org.opensearch.OpenSearchParseException; import org.opensearch.Version; @@ -27,9 +27,9 @@ import java.util.Objects; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.opensearch.action.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.action.pagination.PageParams.PARAM_DESC_SORT_VALUE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; -import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; -import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; public class IndexPaginationStrategyTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/action/pagination/PageParamsTests.java b/server/src/test/java/org/opensearch/action/pagination/PageParamsTests.java new file mode 100644 index 0000000000000..66459ad473a49 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/pagination/PageParamsTests.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +public class PageParamsTests extends OpenSearchTestCase { + + public void testSerialization() throws Exception { + PageParams pageParams = new PageParams("foo", "foo", 1); + try (BytesStreamOutput out = new BytesStreamOutput()) { + pageParams.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertEquals(pageParams, new PageParams(in)); + } + } + } + + public void testSerializationWithRequestedTokenAndSortAbsent() throws Exception { + PageParams pageParams = new PageParams(null, null, 1); + try (BytesStreamOutput out = new BytesStreamOutput()) { + pageParams.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertEquals(pageParams, new PageParams(in)); + } + } + } + +} diff --git a/server/src/test/java/org/opensearch/action/pagination/PageTokenTests.java b/server/src/test/java/org/opensearch/action/pagination/PageTokenTests.java new file mode 100644 index 0000000000000..7c24092dea64c --- /dev/null +++ b/server/src/test/java/org/opensearch/action/pagination/PageTokenTests.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +public class PageTokenTests extends OpenSearchTestCase { + + public void testSerialization() throws Exception { + PageToken pageToken = new PageToken("foo", "test"); + try (BytesStreamOutput out = new BytesStreamOutput()) { + pageToken.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + PageToken deserialized = new PageToken(in); + assertEquals(pageToken, deserialized); + } + } + } + + public void testSerializationWithNextTokenAbsent() throws Exception { + PageToken pageToken = new PageToken(null, "test"); + try (BytesStreamOutput out = new BytesStreamOutput()) { + pageToken.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + PageToken deserialized = new PageToken(in); + assertEquals(pageToken, deserialized); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/action/pagination/ShardPaginationStrategyTests.java b/server/src/test/java/org/opensearch/action/pagination/ShardPaginationStrategyTests.java new file mode 100644 index 0000000000000..aed7315660378 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/pagination/ShardPaginationStrategyTests.java @@ -0,0 +1,527 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.test.OpenSearchTestCase; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.action.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.action.pagination.PageParams.PARAM_DESC_SORT_VALUE; +import static org.opensearch.action.pagination.PaginationStrategy.INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; + +public class ShardPaginationStrategyTests extends OpenSearchTestCase { + + private static final String TEST_INDEX_PREFIX = "test-index-"; + private static final int DEFAULT_NUMBER_OF_SHARDS = 5; + private static final int DEFAULT_NUMBER_OF_REPLICAS = 2; + + /** + * Test validates fetching all the shards for 100 indices in a paginated fashion using {@link ShardPaginationStrategy}, + * while no indices are created or deleted during the fetch. + */ + public void testRetrieveAllShardsWithVaryingPageSize() { + List indexNumberList = new ArrayList<>(); + final int totalIndices = 100; + final int totalShards = totalIndices * DEFAULT_NUMBER_OF_SHARDS * (DEFAULT_NUMBER_OF_REPLICAS + 1); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + // creating a cluster state with 100 indices + Collections.shuffle(indexNumberList, getRandom()); + ClusterState clusterState = getRandomClusterState(indexNumberList); + + // Checking pagination response for different pageSizes, which has a mix of even and odd numbers + // to ensure number of shards in last page is not always equal to pageSize. + List pageSizeList = List.of(3, 7, 10, 13); + + List sortOrderList = List.of(PARAM_ASC_SORT_VALUE, PARAM_DESC_SORT_VALUE); + for (String sortOrder : sortOrderList) { + for (int pageSize : pageSizeList) { + List shardRoutings = new ArrayList<>(); + Set indices = new HashSet<>(); + String requestedToken = null; + // Since each shardID will have number of (replicas + 1) shards, thus number + // of shards in a page should always be a multiple of (replicas + 1). + int totalPagesToFetch = (int) Math.ceil(totalShards * 1.0 / (pageSize - pageSize % (DEFAULT_NUMBER_OF_REPLICAS + 1))); + + long lastPageIndexTime = 0; + for (int pageNum = 1; pageNum <= totalPagesToFetch; pageNum++) { + PageParams pageParams = new PageParams(requestedToken, sortOrder, pageSize); + ShardPaginationStrategy strategy = new ShardPaginationStrategy(pageParams, clusterState); + List pageShards = strategy.getRequestedEntities(); + List pageIndices = strategy.getRequestedIndices(); + if (pageNum < totalPagesToFetch) { + assertNotNull(strategy.getResponseToken().getNextToken()); + } else { + assertNull(strategy.getResponseToken().getNextToken()); + } + shardRoutings.addAll(pageShards); + indices.addAll(pageIndices); + long currentLastIndexTime = clusterState.metadata() + .indices() + .get(pageShards.get(pageShards.size() - 1).getIndexName()) + .getCreationDate(); + if (lastPageIndexTime > 0) { + if (sortOrder.equals(PARAM_ASC_SORT_VALUE)) { + assertTrue(lastPageIndexTime <= currentLastIndexTime); + } else { + assertTrue(lastPageIndexTime >= currentLastIndexTime); + } + } + lastPageIndexTime = currentLastIndexTime; + requestedToken = strategy.getResponseToken().getNextToken(); + } + assertEquals(totalShards, shardRoutings.size()); + assertEquals(totalIndices, indices.size()); + + // verifying the order of shards in the response. + verifyOrderForAllShards(clusterState, shardRoutings, sortOrder); + } + } + } + + /** + * Test validates fetching all the shards (in asc order) for 100 indices in a paginated fashion using {@link ShardPaginationStrategy}. + * While executing queries, it tries to delete an index for which some of the shards have already been fetched along + * with some other indices which are yet to be fetched. It also creates new indices in between the queries. + * Shards corresponding to indices deleted, should not be fetched while for the indices which were newly created, + * should appear in the response. + */ + public void testRetrieveAllShardsInAscOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + List newIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 10; + final int numIndicesToCreate = 5; + final int fixedIndexNumToDelete = 7; + final int numShardsForNewIndices = 4; + final int numReplicasForNewIndices = 5; + final int totalInitialShards = totalIndices * DEFAULT_NUMBER_OF_SHARDS * (DEFAULT_NUMBER_OF_REPLICAS + 1); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 10; + String requestedToken = null; + int numPages = 0; + List shardRoutings = new ArrayList<>(); + Set indicesFetched = new HashSet<>(); + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + ShardPaginationStrategy paginationStrategy = new ShardPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // deleting test-index-7 & 10 more random indices after 11th call. By that time, shards for first 6 + // indices would have been completely fetched. 11th call, would fetch first 3 shardsIDs for test-index-7 and + // if it gets deleted, rest 2 shardIDs for it should not appear. + if (numPages == 11) { + // asserting last elements of current response list, which should be the last shard of test-index-6 + assertEquals(TEST_INDEX_PREFIX + 6, shardRoutings.get(shardRoutings.size() - 1).getIndexName()); + assertEquals(DEFAULT_NUMBER_OF_SHARDS - 1, shardRoutings.get(shardRoutings.size() - 1).getId()); + + // asserting the result of 11th query, which should only contain 3 shardIDs belonging to test-index-7 + assertEquals(3 * (DEFAULT_NUMBER_OF_REPLICAS + 1), paginationStrategy.getRequestedEntities().size()); + assertEquals(1, paginationStrategy.getRequestedIndices().size()); + assertEquals(TEST_INDEX_PREFIX + 7, paginationStrategy.getRequestedIndices().get(0)); + assertEquals(2, paginationStrategy.getRequestedEntities().get(8).id()); + + clusterState = deleteIndexFromClusterState(clusterState, fixedIndexNumToDelete); + deletedIndices = indexNumberList.subList(20, indexNumberList.size()); + Collections.shuffle(deletedIndices, getRandom()); + deletedIndices = deletedIndices.subList(0, numIndicesToDelete); + for (Integer index : deletedIndices) { + clusterState = deleteIndexFromClusterState(clusterState, index); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + newIndices.add(TEST_INDEX_PREFIX + indexNumber); + clusterState = addIndexToClusterState(clusterState, indexNumber, numShardsForNewIndices, numReplicasForNewIndices); + } + } + assertTrue(paginationStrategy.getRequestedEntities().size() <= pageSize); + shardRoutings.addAll(paginationStrategy.getRequestedEntities()); + indicesFetched.addAll(paginationStrategy.getRequestedIndices()); + } while (Objects.nonNull(requestedToken)); + + assertEquals(totalIndices + numIndicesToCreate - numIndicesToDelete, indicesFetched.size()); + // finalTotalShards = InitialTotal - 2ShardIdsForIndex7 - ShardsFor10RandomlyDeletedIndices + ShardsForNewlyCreatedIndices + final int totalShards = totalInitialShards - 2 * (DEFAULT_NUMBER_OF_REPLICAS + 1) - numIndicesToDelete * DEFAULT_NUMBER_OF_SHARDS + * (DEFAULT_NUMBER_OF_REPLICAS + 1) + numIndicesToCreate * numShardsForNewIndices * (numReplicasForNewIndices + 1); + assertEquals(totalShards, shardRoutings.size()); + + // deleted test-index-7, should appear in the response shards and indices + assertTrue(indicesFetched.contains(TEST_INDEX_PREFIX + 7)); + assertEquals( + shardRoutings.stream().filter(shard -> shard.getIndexName().equals(TEST_INDEX_PREFIX + 7)).count(), + 3 * (DEFAULT_NUMBER_OF_REPLICAS + 1) + ); + + // none of the randomly deleted index should appear in the list of fetched indices + for (Integer index : deletedIndices) { + String indexName = TEST_INDEX_PREFIX + index; + assertFalse(indicesFetched.contains(indexName)); + assertEquals(shardRoutings.stream().filter(shard -> shard.getIndexName().equals(indexName)).count(), 0); + } + + // all the newly created indices should be present in the list of fetched indices + verifyOrderForAllShards( + clusterState, + shardRoutings.stream().filter(shard -> newIndices.contains(shard.getIndexName())).collect(Collectors.toList()), + PARAM_ASC_SORT_VALUE + ); + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + String indexName = TEST_INDEX_PREFIX + indexNumber; + assertTrue(indicesFetched.contains(indexName)); + assertEquals( + numShardsForNewIndices * (numReplicasForNewIndices + 1), + shardRoutings.stream().filter(shard -> shard.getIndexName().equals(indexName)).count() + ); + } + } + + /** + * Test validates fetching all the shards (in desc order) for 100 indices in a paginated fashion using {@link ShardPaginationStrategy}. + * While executing queries, it tries to delete an index for which some of the shards have already been fetched along + * with some other indices which are yet to be fetched. It also creates new indices in between the queries. + * Shards corresponding to indices deleted, should not be fetched while for the indices which were newly created, + * should appear in the response. + */ + public void testRetrieveAllShardsInDescOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 10; + final int numIndicesToCreate = 5; + final int fixedIndexNumToDelete = 94; + final int numShardsForNewIndices = 4; + final int numReplicasForNewIndices = 5; + final int totalInitialShards = totalIndices * DEFAULT_NUMBER_OF_SHARDS * (DEFAULT_NUMBER_OF_REPLICAS + 1); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 10; + String requestedToken = null; + int numPages = 0; + List shardRoutings = new ArrayList<>(); + Set indicesFetched = new HashSet<>(); + do { + PageParams pageParams = new PageParams(requestedToken, PARAM_DESC_SORT_VALUE, pageSize); + ShardPaginationStrategy paginationStrategy = new ShardPaginationStrategy(pageParams, clusterState); + numPages++; + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // deleting test-index-94 & 10 more random indices after 11th call. By that time, shards for last 6 + // indices would have been completely fetched. 11th call, would fetch first 3 shardsIDs for test-index-94 and + // if it gets deleted, rest 2 shardIDs for it should not appear. + if (numPages == 11) { + // asserting last elements of current response list, which should be the last shard of test-index-95 + assertEquals(TEST_INDEX_PREFIX + 95, shardRoutings.get(shardRoutings.size() - 1).getIndexName()); + assertEquals(DEFAULT_NUMBER_OF_SHARDS - 1, shardRoutings.get(shardRoutings.size() - 1).getId()); + + // asserting the result of 11th query, which should only contain 3 shardIDs belonging to test-index-7 + assertEquals(3 * (DEFAULT_NUMBER_OF_REPLICAS + 1), paginationStrategy.getRequestedEntities().size()); + assertEquals(1, paginationStrategy.getRequestedIndices().size()); + assertEquals(TEST_INDEX_PREFIX + 94, paginationStrategy.getRequestedIndices().get(0)); + assertEquals(2, paginationStrategy.getRequestedEntities().get(8).id()); + + clusterState = deleteIndexFromClusterState(clusterState, fixedIndexNumToDelete); + deletedIndices = indexNumberList.subList(0, 80); + Collections.shuffle(deletedIndices, getRandom()); + deletedIndices = deletedIndices.subList(0, numIndicesToDelete); + for (Integer index : deletedIndices) { + clusterState = deleteIndexFromClusterState(clusterState, index); + } + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber, numShardsForNewIndices, numReplicasForNewIndices); + } + } + assertTrue(paginationStrategy.getRequestedEntities().size() <= pageSize); + shardRoutings.addAll(paginationStrategy.getRequestedEntities()); + indicesFetched.addAll(paginationStrategy.getRequestedIndices()); + } while (Objects.nonNull(requestedToken)); + assertEquals(totalIndices - numIndicesToDelete, indicesFetched.size()); + // finalTotalShards = InitialTotal - 2ShardIdsForIndex7 - ShardsFor10RandomlyDeletedIndices + final int totalShards = totalInitialShards - 2 * (DEFAULT_NUMBER_OF_REPLICAS + 1) - numIndicesToDelete * DEFAULT_NUMBER_OF_SHARDS + * (DEFAULT_NUMBER_OF_REPLICAS + 1); + assertEquals(totalShards, shardRoutings.size()); + + // deleted test-index-94, should appear in the response shards and indices + assertTrue(indicesFetched.contains(TEST_INDEX_PREFIX + fixedIndexNumToDelete)); + assertEquals( + shardRoutings.stream().filter(shard -> shard.getIndexName().equals(TEST_INDEX_PREFIX + fixedIndexNumToDelete)).count(), + 3 * (DEFAULT_NUMBER_OF_REPLICAS + 1) + ); + + // none of the randomly deleted index should appear in the list of fetched indices + for (int deletedIndex : deletedIndices) { + String indexName = TEST_INDEX_PREFIX + deletedIndex; + assertFalse(indicesFetched.contains(indexName)); + assertEquals(shardRoutings.stream().filter(shard -> shard.getIndexName().equals(indexName)).count(), 0); + } + + // none of the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + String indexName = TEST_INDEX_PREFIX + indexNumber; + assertFalse(indicesFetched.contains(indexName)); + assertEquals(0, shardRoutings.stream().filter(shard -> shard.getIndexName().equals(indexName)).count()); + } + } + + /** + * Validates strategy fails with IllegalArgumentException when requests size in pageParam is smaller + * than #(replicas + 1) for any of the index. + */ + public void testIllegalSizeArgumentRequestedFromStrategy() { + int numIndices = 6; + int numShards = 5; + int numReplicas = 8; + int pageSize = numReplicas + 1; + ClusterState clusterState = getRandomClusterState(Collections.emptyList()); + for (int index = 1; index < numIndices; index++) { + clusterState = addIndexToClusterState(clusterState, index, numShards, numReplicas); + } + clusterState = addIndexToClusterState(clusterState, numIndices, numShards + 1, numReplicas + 1); + + try { + String requestedToken = null; + ShardPaginationStrategy strategy; + do { + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + strategy = new ShardPaginationStrategy(pageParams, clusterState); + requestedToken = strategy.getResponseToken().getNextToken(); + } while (requestedToken != null); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("size value should be greater than the replica count of all indices")); + } + } + + public void testRetrieveShardsWhenLastIndexGetsDeletedAndReCreated() { + final int totalIndices = 3; + final int numShards = 3; + final int numReplicas = 3; + final int pageSize = (numReplicas + 1) * 2; + final int totalPages = (int) Math.ceil((totalIndices * numShards * (numReplicas + 1) * 1.0) / pageSize); + ClusterState clusterState = getRandomClusterState(Collections.emptyList()); + for (int indexNum = 1; indexNum <= totalIndices; indexNum++) { + clusterState = addIndexToClusterState(clusterState, indexNum, numShards, numReplicas); + } + + String requestedToken = null; + ShardPaginationStrategy strategy = null; + for (int page = 1; page < totalPages; page++) { + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + strategy = new ShardPaginationStrategy(pageParams, clusterState); + requestedToken = strategy.getResponseToken().getNextToken(); + } + // The last page was supposed to fetch the remaining two indices for test-index-3 + assertEquals(8, strategy.getRequestedEntities().size()); + assertEquals(1, strategy.getRequestedIndices().size()); + assertEquals(TEST_INDEX_PREFIX + 3, strategy.getRequestedIndices().get(0)); + + // Delete the index and re-create + clusterState = deleteIndexFromClusterState(clusterState, 3); + clusterState = addIndexToClusterState( + clusterState, + 3, + numShards, + numReplicas, + Instant.now().plus(4, ChronoUnit.SECONDS).toEpochMilli() + ); + + // since test-index-3 should now be considered a new index, all shards for it should appear + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + strategy = new ShardPaginationStrategy(pageParams, clusterState); + assertEquals(8, strategy.getRequestedEntities().size()); + for (ShardRouting shardRouting : strategy.getRequestedEntities()) { + assertTrue( + (shardRouting.getId() == 0 || shardRouting.getId() == 1) + && Objects.equals(shardRouting.getIndexName(), TEST_INDEX_PREFIX + 3) + ); + } + assertEquals(1, strategy.getRequestedIndices().size()); + assertEquals(TEST_INDEX_PREFIX + 3, strategy.getRequestedIndices().get(0)); + assertNotNull(strategy.getResponseToken().getNextToken()); + + requestedToken = strategy.getResponseToken().getNextToken(); + pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + strategy = new ShardPaginationStrategy(pageParams, clusterState); + assertEquals(4, strategy.getRequestedEntities().size()); + for (ShardRouting shardRouting : strategy.getRequestedEntities()) { + assertTrue(shardRouting.getId() == 2 && Objects.equals(shardRouting.getIndexName(), TEST_INDEX_PREFIX + 3)); + } + assertEquals(1, strategy.getRequestedIndices().size()); + assertEquals(TEST_INDEX_PREFIX + 3, strategy.getRequestedIndices().get(0)); + assertNull(strategy.getResponseToken().getNextToken()); + } + + public void testCreatingShardStrategyPageTokenWithRequestedTokenNull() { + try { + new ShardPaginationStrategy.ShardStrategyToken(null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("requestedTokenString can not be null"); + } + } + + public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { + assertThrows(OpenSearchParseException.class, () -> new ShardPaginationStrategy.ShardStrategyToken("3%4%5")); + } + + public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new ShardPaginationStrategy.ShardStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543")) + ); + assertThrows( + OpenSearchParseException.class, + () -> new ShardPaginationStrategy.ShardStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) + ); + } + + public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new ShardPaginationStrategy.ShardStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|1725361543|index")) + ); + } + + public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { + try { + new ShardPaginationStrategy.ShardStrategyToken(null, 0, 1234l); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("index name should be provided"); + } + } + + public void testCreatingIndexStrategyPageTokenWithNonParseableShardID() { + try { + new ShardPaginationStrategy.ShardStrategyToken(PaginationStrategy.encryptStringToken("shardID|1725361543|index")); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + + /** + * @param indexNumbers would be used to create indices having names with integer appended after foo, like foo1, foo2. + * @return random clusterState consisting of indices having their creation times set to the integer used to name them. + */ + private ClusterState getRandomClusterState(List indexNumbers) { + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().build()) + .routingTable(RoutingTable.builder().build()) + .build(); + for (Integer indexNumber : indexNumbers) { + clusterState = addIndexToClusterState(clusterState, indexNumber, DEFAULT_NUMBER_OF_SHARDS, DEFAULT_NUMBER_OF_REPLICAS); + } + return clusterState; + } + + private ClusterState addIndexToClusterState( + ClusterState clusterState, + final int indexNumber, + final int numShards, + final int numReplicas + ) { + return addIndexToClusterState( + clusterState, + indexNumber, + numShards, + numReplicas, + Instant.now().plus(indexNumber, ChronoUnit.SECONDS).toEpochMilli() + ); + } + + private ClusterState addIndexToClusterState( + ClusterState clusterState, + final int indexNumber, + final int numShards, + final int numReplicas, + final long creationTime + ) { + IndexMetadata indexMetadata = IndexMetadata.builder(TEST_INDEX_PREFIX + indexNumber) + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, creationTime)) + .numberOfShards(numShards) + .numberOfReplicas(numReplicas) + .build(); + IndexRoutingTable.Builder indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()).initializeAsNew( + indexMetadata + ); + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder(clusterState.routingTable()).add(indexRoutingTableBuilder).build()) + .build(); + } + + private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int indexNumber) { + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).remove(TEST_INDEX_PREFIX + indexNumber)) + .routingTable(RoutingTable.builder(clusterState.routingTable()).remove(TEST_INDEX_PREFIX + indexNumber).build()) + .build(); + } + + private void verifyOrderForAllShards(ClusterState clusterState, List shardRoutings, String sortOrder) { + ShardRouting prevShard = shardRoutings.get(0); + for (ShardRouting shard : shardRoutings.subList(1, shardRoutings.size())) { + if (Objects.equals(shard.getIndexName(), prevShard.getIndexName())) { + assertTrue(shard.getId() >= prevShard.getId()); + } else { + if (sortOrder.equals(PARAM_ASC_SORT_VALUE)) { + assertTrue( + clusterState.metadata().index(shard.getIndexName()).getCreationDate() > clusterState.metadata() + .index(prevShard.getIndexName()) + .getCreationDate() + ); + } else { + assertTrue( + clusterState.metadata().index(shard.getIndexName()).getCreationDate() < clusterState.metadata() + .index(prevShard.getIndexName()) + .getCreationDate() + ); + } + prevShard = shard; + } + } + } + +} diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 2debb00a0d14f..67baedb7df5a9 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexStats; +import org.opensearch.action.pagination.PageToken; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; @@ -50,7 +51,6 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; import org.opensearch.rest.action.list.RestIndicesListAction; -import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java index 883df7da5d717..c412167a10c75 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java @@ -38,6 +38,7 @@ import org.opensearch.action.admin.indices.stats.IndexStats; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.action.pagination.PageToken; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -50,6 +51,7 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Before; import java.nio.file.Path; import java.util.ArrayList; @@ -64,14 +66,18 @@ public class RestShardsActionTests extends OpenSearchTestCase { - public void testBuildTable() { + private final DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + private List shardRoutings = new ArrayList<>(); + private Map shardStatsMap = new HashMap<>(); + private ClusterStateResponse state; + private IndicesStatsResponse stats; + + @Before + public void setup() { final int numShards = randomIntBetween(1, 5); long numDocs = randomLongBetween(0, 10000); long numDeletedDocs = randomLongBetween(0, 100); - DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); - List shardRoutings = new ArrayList<>(numShards); - Map shardStatsMap = new HashMap<>(); String index = "index"; for (int i = 0; i < numShards; i++) { ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); @@ -97,23 +103,49 @@ public void testBuildTable() { when(indexStats.getPrimaries()).thenReturn(new CommonStats()); when(indexStats.getTotal()).thenReturn(new CommonStats()); - IndicesStatsResponse stats = mock(IndicesStatsResponse.class); + stats = mock(IndicesStatsResponse.class); when(stats.asMap()).thenReturn(shardStatsMap); DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(discoveryNodes.get(localNode.getId())).thenReturn(localNode); - ClusterStateResponse state = mock(ClusterStateResponse.class); + state = mock(ClusterStateResponse.class); RoutingTable routingTable = mock(RoutingTable.class); when(routingTable.allShards()).thenReturn(shardRoutings); ClusterState clusterState = mock(ClusterState.class); when(clusterState.routingTable()).thenReturn(routingTable); when(clusterState.nodes()).thenReturn(discoveryNodes); when(state.getState()).thenReturn(clusterState); + } + + public void testBuildTable() { + final RestShardsAction action = new RestShardsAction(); + final Table table = action.buildTable( + new FakeRestRequest(), + state.getState().nodes(), + stats, + state.getState().routingTable().allShards(), + null + ); + assertTable(table); + } + public void testBuildTableWithPageToken() { final RestShardsAction action = new RestShardsAction(); - final Table table = action.buildTable(new FakeRestRequest(), state, stats); + final Table table = action.buildTable( + new FakeRestRequest(), + state.getState().nodes(), + stats, + state.getState().routingTable().allShards(), + new PageToken("foo", "test") + ); + assertTable(table); + assertNotNull(table.getPageToken()); + assertEquals("foo", table.getPageToken().getNextToken()); + assertEquals("test", table.getPageToken().getPaginatedEntity()); + } + private void assertTable(Table table) { // now, verify the table is correct List headers = table.getHeaders(); assertThat(headers.get(0).value, equalTo("index")); @@ -128,7 +160,7 @@ public void testBuildTable() { assertThat(headers.get(79).value, equalTo("docs.deleted")); final List> rows = table.getRows(); - assertThat(rows.size(), equalTo(numShards)); + assertThat(rows.size(), equalTo(shardRoutings.size())); Iterator shardRoutingsIt = shardRoutings.iterator(); for (final List row : rows) { diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java index a82e563d70273..f475d0ff1efda 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java @@ -32,12 +32,12 @@ package org.opensearch.rest.action.cat; +import org.opensearch.action.pagination.PageToken; import org.opensearch.common.Table; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/rest/action/list/RestShardsListActionTests.java b/server/src/test/java/org/opensearch/rest/action/list/RestShardsListActionTests.java new file mode 100644 index 0000000000000..cebcee05ca69b --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/action/list/RestShardsListActionTests.java @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.action.pagination.PageParams; +import org.opensearch.action.pagination.PaginationStrategy; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.action.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.action.list.RestShardsListAction.MAX_SUPPORTED_LIST_SHARDS_PAGE_SIZE; +import static org.opensearch.rest.action.list.RestShardsListAction.MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE; + +public class RestShardsListActionTests extends OpenSearchTestCase { + + private final RestShardsListAction action = new RestShardsListAction(); + + public void testShardsListActionIsPaginated() { + assertTrue(action.isActionPaginated()); + } + + public void testValidateAndGetPageParamsWithDefaultParams() { + Map params = new HashMap<>(); + RestRequest restRequest = new FakeRestRequest(params); + PageParams pageParams = action.validateAndGetPageParams(restRequest); + assertEquals(MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE, pageParams.getSize()); + assertEquals(PARAM_ASC_SORT_VALUE, pageParams.getSort()); + assertNull(pageParams.getRequestedToken()); + } + + public void testValidateAndGetPageParamsWithSizeBelowMin() { + Map params = new HashMap<>(); + params.put("size", String.valueOf(MIN_SUPPORTED_LIST_SHARDS_PAGE_SIZE - 1)); + RestRequest restRequest = new FakeRestRequest(params); + assertThrows(IllegalArgumentException.class, () -> action.validateAndGetPageParams(restRequest)); + } + + public void testValidateAndGetPageParamsWithSizeAboveRange() { + Map params = new HashMap<>(); + params.put("size", String.valueOf(MAX_SUPPORTED_LIST_SHARDS_PAGE_SIZE * 10)); + RestRequest restRequest = new FakeRestRequest(params); + assertThrows(IllegalArgumentException.class, () -> action.validateAndGetPageParams(restRequest)); + } + + public void testValidateAndGetPageParamsWithInvalidRequestToken() { + Map params = new HashMap<>(); + params.put("next_token", PaginationStrategy.encryptStringToken("1|-1|test")); + RestRequest restRequest = new FakeRestRequest(params); + assertThrows(OpenSearchParseException.class, () -> action.validateAndGetPageParams(restRequest)); + } + + public void testValidateAndGetPageParamsWithValidPageParams() { + Map params = new HashMap<>(); + params.put("next_token", PaginationStrategy.encryptStringToken("1|1|test")); + params.put("sort", "asc"); + params.put("size", "3000"); + RestRequest restRequest = new FakeRestRequest(params); + PageParams pageParams = action.validateAndGetPageParams(restRequest); + + assertEquals(PaginationStrategy.encryptStringToken("1|1|test"), pageParams.getRequestedToken()); + assertEquals(3000, pageParams.getSize()); + assertEquals("asc", pageParams.getSort()); + } + +} diff --git a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java index e7810ae4c8f1c..1880977c2d35d 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java @@ -60,6 +60,15 @@ public FakeRestRequest() { ); } + public FakeRestRequest(Map params) { + this( + NamedXContentRegistry.EMPTY, + new FakeHttpRequest(Method.GET, "", BytesArray.EMPTY, new HashMap<>()), + params, + new FakeHttpChannel(null) + ); + } + private FakeRestRequest( NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, From 146b0f7264b1426d475e757729d213097b776212 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 7 Oct 2024 08:26:43 -0400 Subject: [PATCH 192/357] Update Apache Lucene to 9.12.0 (#15333) * Update Apache Lucene to 9.12.0 Signed-off-by: Andriy Redko * change to IOContext READONCE in locations where the file is not expected to be read multiple times. Signed-off-by: Marc Handalian * Use READ IOContext for all non Segment* files when copying node-node Signed-off-by: Marc Handalian * Fixing more test failures Signed-off-by: Andriy Redko * Move Composite912Codec under org.opensearch.index.codec.composite.composite912 package Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko Signed-off-by: Marc Handalian Co-authored-by: Marc Handalian --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- ...cene-core-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../core/licenses/lucene-core-9.12.0.jar.sha1 | 1 + ...pressions-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-expressions-9.12.0.jar.sha1 | 1 + ...lysis-icu-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-icu-9.12.0.jar.sha1 | 1 + ...-kuromoji-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.12.0.jar.sha1 | 1 + ...ysis-nori-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-nori-9.12.0.jar.sha1 | 1 + ...-phonetic-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.12.0.jar.sha1 | 1 + ...s-smartcn-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.12.0.jar.sha1 | 1 + ...s-stempel-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-stempel-9.12.0.jar.sha1 | 1 + ...orfologik-9.12.0-snapshot-847316d.jar.sha1 | 1 - ...lucene-analysis-morfologik-9.12.0.jar.sha1 | 1 + .../index/codec/CorrelationCodecVersion.java | 2 +- ...is-common-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-analysis-common-9.12.0.jar.sha1 | 1 + ...rd-codecs-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-backward-codecs-9.12.0.jar.sha1 | 1 + ...cene-core-9.12.0-snapshot-847316d.jar.sha1 | 1 - server/licenses/lucene-core-9.12.0.jar.sha1 | 1 + ...-grouping-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../licenses/lucene-grouping-9.12.0.jar.sha1 | 1 + ...ghlighter-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-highlighter-9.12.0.jar.sha1 | 1 + ...cene-join-9.12.0-snapshot-847316d.jar.sha1 | 1 - server/licenses/lucene-join-9.12.0.jar.sha1 | 1 + ...ne-memory-9.12.0-snapshot-847316d.jar.sha1 | 1 - server/licenses/lucene-memory-9.12.0.jar.sha1 | 1 + ...cene-misc-9.12.0-snapshot-847316d.jar.sha1 | 1 - server/licenses/lucene-misc-9.12.0.jar.sha1 | 1 + ...e-queries-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../licenses/lucene-queries-9.12.0.jar.sha1 | 1 + ...eryparser-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-queryparser-9.12.0.jar.sha1 | 1 + ...e-sandbox-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../licenses/lucene-sandbox-9.12.0.jar.sha1 | 1 + ...al-extras-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../lucene-spatial-extras-9.12.0.jar.sha1 | 1 + ...spatial3d-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../licenses/lucene-spatial3d-9.12.0.jar.sha1 | 1 + ...e-suggest-9.12.0-snapshot-847316d.jar.sha1 | 1 - .../licenses/lucene-suggest-9.12.0.jar.sha1 | 1 + .../node/tasks/ConcurrentSearchTasksIT.java | 2 +- .../recovery/TruncatedRecoveryIT.java | 3 +- .../similarity/LegacyBM25Similarity.java | 10 +--- .../apache/lucene/util/CombinedBitSet.java | 26 ++++++--- .../org/opensearch/common/lucene/Lucene.java | 4 +- .../opensearch/index/codec/CodecService.java | 12 ++-- .../PerFieldMappingPostingFormatCodec.java | 4 +- .../composite/CompositeCodecFactory.java | 11 ++-- .../composite912/Composite912Codec.java | 58 +++++++++++++++++++ .../composite/composite912/package-info.java | 12 ++++ .../composite99/Composite99Codec.java | 8 +-- .../index/mapper/CompletionFieldMapper.java | 4 +- .../opensearch/index/shard/IndexShard.java | 4 +- .../shard/RemoteStoreRefreshListener.java | 4 +- .../index/similarity/SimilarityProviders.java | 3 +- .../index/store/RemoteDirectory.java | 2 +- .../store/RemoteSegmentStoreDirectory.java | 2 +- .../org/opensearch/index/store/Store.java | 8 +-- .../SegmentFileTransferHandler.java | 39 ++++++++++--- .../replication/SegmentReplicationTarget.java | 2 +- .../blobstore/BlobStoreRepository.java | 2 +- .../search/internal/ContextIndexSearcher.java | 3 +- .../services/org.apache.lucene.codecs.Codec | 1 + .../opensearch/index/codec/CodecTests.java | 40 ++++++------- .../LuceneDocValuesConsumerFactoryTests.java | 6 +- .../LuceneDocValuesProducerFactoryTests.java | 6 +- .../StarTreeDocValuesFormatTests.java | 8 +-- .../builder/AbstractStarTreeBuilderTests.java | 12 ++-- .../builder/BaseStarTreeBuilderTests.java | 6 +- .../builder/StarTreesBuilderTests.java | 6 +- .../meta/StarTreeMetadataTests.java | 6 +- .../engine/CompletionStatsCacheTests.java | 8 +-- .../RemoteIndexShardCorruptionTests.java | 2 +- .../opensearch/index/store/StoreTests.java | 2 +- .../org/opensearch/test/CorruptionUtils.java | 2 +- 84 files changed, 231 insertions(+), 148 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.12.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-core-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-join-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.12.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.12.0.jar.sha1 create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java create mode 100644 server/src/main/java/org/opensearch/index/codec/composite/composite912/package-info.java rename server/src/test/java/org/opensearch/index/codec/{composite99 => composite912}/datacube/startree/StarTreeDocValuesFormatTests.java (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbf19fb0d35be..2e3a515fb8008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) +- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333)) - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 8cede6b717883..19e2ba2e2c5bb 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.12.0-snapshot-847316d +lucene = 9.12.0 bundled_jdk_vendor = adoptium bundled_jdk = 21.0.4+7 diff --git a/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index e3fd1708ea428..0000000000000 --- a/libs/core/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -51ff4940eb1024184bbaa5dae39695d2392c5bab \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.12.0.jar.sha1 b/libs/core/licenses/lucene-core-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..e55f896dedb63 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.12.0.jar.sha1 @@ -0,0 +1 @@ +fdb055d569bb20bfce9618fe2b01c29bab7f290c \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 83dd8e657bdd5..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b866103bbaca4141c152deca9252bd137026dafc \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..476049a66cc08 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1 @@ -0,0 +1 @@ +5ba843374a0aab3dfe0b11cb28b251844d85bf5b \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 80e254ed3d098..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04436942995a4952ce5654126dfb767d6335674e \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..31398b27708a3 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1 @@ -0,0 +1 @@ +a9232b6a4882979118d3281b98dfdb6e0e1cb5ca \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 3baed2a6e660b..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85918e24fc3bf63fcd953807ab2eb3fa55c987c2 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..fa4c9d2d09d6e --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1 @@ -0,0 +1 @@ +a3a6950ffc22e76a082e1b3cefb022b9f7870d29 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 4e9327112d412..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -15e425e9cc0ab9d65fac3c919199a24dfa3631eb \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..576b924286d2d --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1 @@ -0,0 +1 @@ +e71f85b72ed3939039ba8897b28b065dd11918b9 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 7e7e9fe5b22b4..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d16c18348e7d4a00cb83100c43f3e21239d224e \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..c8c146bbd0d25 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1 @@ -0,0 +1 @@ +6baa3ae7ab20d6e644cf0bedb271c50a44c0e259 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 98e0ecc9cbb89..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2ef6d9dffc6816d3cd04a54fe1ee43e13f850a37 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..54ea0b19f2a7b --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1 @@ -0,0 +1 @@ +f183e1e8b1eaaa4dec444774a285bb8b66518522 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index ef675f2b9702e..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e72b2262f5393d9ff255fb901297d4e6790e9102 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..5442a40f5bba2 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1 @@ -0,0 +1 @@ +b81a609934e65d12ab9d2d84bc2ea6f56a360e57 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index d8bbac27fd360..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -416ac44b2e76592c9e85338798cae93c3cf5475e \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..60fd4015cfde0 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1 @@ -0,0 +1 @@ +bec069f286b45f20b743c81e84202369cd0467e7 \ No newline at end of file diff --git a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java index 3fcc995fb4199..9dbb695f14b78 100644 --- a/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java +++ b/plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java @@ -8,8 +8,8 @@ package org.opensearch.plugin.correlation.core.index.codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.index.mapper.MapperService; import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec; import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat; diff --git a/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index f1249066d10f2..0000000000000 --- a/server/licenses/lucene-analysis-common-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e282aab7388efc911348f1eacd90e661580dda7 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.12.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..fd952034f3742 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.12.0.jar.sha1 @@ -0,0 +1 @@ +4c2503cfaba37249e20ea877555cb52ee89d1ae1 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index ac50c5e110a72..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -69e59ba4bed4c58836d2727d72b7f0095d2dcb92 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.12.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..2993134edd610 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.12.0.jar.sha1 @@ -0,0 +1 @@ +68fe98c94e9644a584ea1bf525e68d9406fc61ec \ No newline at end of file diff --git a/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index e3fd1708ea428..0000000000000 --- a/server/licenses/lucene-core-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -51ff4940eb1024184bbaa5dae39695d2392c5bab \ No newline at end of file diff --git a/server/licenses/lucene-core-9.12.0.jar.sha1 b/server/licenses/lucene-core-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..e55f896dedb63 --- /dev/null +++ b/server/licenses/lucene-core-9.12.0.jar.sha1 @@ -0,0 +1 @@ +fdb055d569bb20bfce9618fe2b01c29bab7f290c \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index cc5bf5bfd8ec0..0000000000000 --- a/server/licenses/lucene-grouping-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5847a7d47f13ecb7f039fb9adf6f3b8e4bddde77 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.12.0.jar.sha1 b/server/licenses/lucene-grouping-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..48388974bb38f --- /dev/null +++ b/server/licenses/lucene-grouping-9.12.0.jar.sha1 @@ -0,0 +1 @@ +ccf99f8db57aa97b2c1f95c5cc2a11156a043921 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index eb14059d2cd8c..0000000000000 --- a/server/licenses/lucene-highlighter-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7cc0a26777a479f06fbcfae7abc23e784e1a00dc \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.12.0.jar.sha1 b/server/licenses/lucene-highlighter-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..3d457579da892 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.12.0.jar.sha1 @@ -0,0 +1 @@ +e93429f66fbcd3b58d81f01223d6ce5688047296 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index b87170c39c78c..0000000000000 --- a/server/licenses/lucene-join-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9cd99401c826d910da3c2beab8e42f1af8be6ea4 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.12.0.jar.sha1 b/server/licenses/lucene-join-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..c5f6d16598a60 --- /dev/null +++ b/server/licenses/lucene-join-9.12.0.jar.sha1 @@ -0,0 +1 @@ +14c802d6955eaf11107375a2ada8fe8ec53b3e01 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index de591dd659cb5..0000000000000 --- a/server/licenses/lucene-memory-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cfee136ecbc3df7adc38b38e020dca5e61c22773 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.12.0.jar.sha1 b/server/licenses/lucene-memory-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..e7ac44089c006 --- /dev/null +++ b/server/licenses/lucene-memory-9.12.0.jar.sha1 @@ -0,0 +1 @@ +ffe090339540876b40df792aee51a42af6b3f37f \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 1a999bb9c6686..0000000000000 --- a/server/licenses/lucene-misc-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afbc5adf93d4eb1a1b109ad828d1968bf16ef292 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.12.0.jar.sha1 b/server/licenses/lucene-misc-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..afb546be4e032 --- /dev/null +++ b/server/licenses/lucene-misc-9.12.0.jar.sha1 @@ -0,0 +1 @@ +ad17704ee90eb926b6d3105f7027485cdadbecd9 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 783a26551ae8c..0000000000000 --- a/server/licenses/lucene-queries-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -16907c36f6adb8ba8f260e05738c66afb37c72d3 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.12.0.jar.sha1 b/server/licenses/lucene-queries-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..e24756e38dad2 --- /dev/null +++ b/server/licenses/lucene-queries-9.12.0.jar.sha1 @@ -0,0 +1 @@ +3ac2a62b0b55c5725bb65f0c5454f9f8a401cf43 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index b3e9e4de96174..0000000000000 --- a/server/licenses/lucene-queryparser-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -72baa9bddcf2efb71ffb695f1e9f548699ec13a0 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.12.0.jar.sha1 b/server/licenses/lucene-queryparser-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..e93e00a063dd0 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.12.0.jar.sha1 @@ -0,0 +1 @@ +55959399373876f4c184944315458dc6b88fbd81 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 2aefa435b1e9a..0000000000000 --- a/server/licenses/lucene-sandbox-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dd3c63066f583d90b563ebaa6fbe61c603403acb \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.12.0.jar.sha1 b/server/licenses/lucene-sandbox-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..a3fd8446e0dbc --- /dev/null +++ b/server/licenses/lucene-sandbox-9.12.0.jar.sha1 @@ -0,0 +1 @@ +f65882536d681c11a1cbc920e5679201101e3603 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index d27112c6db6ab..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -69b99530e0b05251c12863bee6a9325cafd5fdaa \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.12.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..b0f11fb667faf --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.12.0.jar.sha1 @@ -0,0 +1 @@ +9d00cc7cc2279822ef6740f0677cafacfb439fa8 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 29423ac0ababd..0000000000000 --- a/server/licenses/lucene-spatial3d-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a67d193b4b08790169db7cf005a2429991260287 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.12.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..858eee25ac191 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.12.0.jar.sha1 @@ -0,0 +1 @@ +e3092632ca1d4427d3ebb2c866ac89d90f5b61ec \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 b/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 deleted file mode 100644 index 6ce1f639ccbb7..0000000000000 --- a/server/licenses/lucene-suggest-9.12.0-snapshot-847316d.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a1625ae39071ccbfb3af11df5a74291758f4b47 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.12.0.jar.sha1 b/server/licenses/lucene-suggest-9.12.0.jar.sha1 new file mode 100644 index 0000000000000..973a7726d845d --- /dev/null +++ b/server/licenses/lucene-suggest-9.12.0.jar.sha1 @@ -0,0 +1 @@ +e1c6636499317ebe498f3490a1ec8b86b8a363dd \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java index 8b3c40c43e2d2..e2a86b9358829 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java @@ -110,7 +110,7 @@ public void testConcurrentSearchTaskTracking() { Map> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId()); // Concurrent search forks each slice of 5 segments to different thread (see please // https://github.com/apache/lucene/issues/12498) - assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size()); + assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size()); // assert that all task descriptions have non-zero length MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0)); diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java index 692beb86279b9..e9515700199ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java @@ -58,6 +58,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING; @@ -184,7 +185,7 @@ public void testCancelRecoveryAndResume() throws Exception { ) .get(); - latch.await(); + latch.await(5, TimeUnit.MINUTES); // at this point we got some truncated left overs on the replica on the unlucky node // now we are allowing the recovery to allocate again and finish to see if we wipe the truncated files diff --git a/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java index 3b812e1c70368..e48a6704ec217 100644 --- a/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java +++ b/server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java @@ -70,6 +70,7 @@ public LegacyBM25Similarity(float k1, float b) { * within the range {@code [0..1]} */ public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) { + super(discountOverlaps); this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps); } @@ -101,15 +102,6 @@ public final float getB() { return bm25Similarity.getB(); } - /** - * Returns true if overlap tokens are discounted from the document's length. - * - * @see #LegacyBM25Similarity(float, float, boolean) - */ - public boolean getDiscountOverlaps() { - return bm25Similarity.getDiscountOverlaps(); - } - @Override public String toString() { return bm25Similarity.toString(); diff --git a/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java b/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java index 1b4f31892c7f8..027a7ed292863 100644 --- a/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java +++ b/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java @@ -87,15 +87,7 @@ public int prevSetBit(int index) { @Override public int nextSetBit(int index) { - assert index >= 0 && index < length : "index=" + index + " numBits=" + length(); - int next = first.nextSetBit(index); - while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) { - if (next == length() - 1) { - return DocIdSetIterator.NO_MORE_DOCS; - } - next = first.nextSetBit(next + 1); - } - return next; + return nextSetBit(index, length() - 1); } @Override @@ -132,4 +124,20 @@ public void clear(int startIndex, int endIndex) { public boolean getAndSet(int i) { throw new UnsupportedOperationException("not implemented"); } + + @Override + public int nextSetBit(int start, int end) { + assert start >= 0 && start < length() : "start=" + start + " numBits=" + length(); + assert end >= start && end < length() : "end=" + end + " numBits=" + length(); + + int next = first.nextSetBit(start); + while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) { + if (next >= end) { + return DocIdSetIterator.NO_MORE_DOCS; + } + next = first.nextSetBit(next + 1); + } + return next; + + } } diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 2c7b6b552b43f..361ee859e3e6a 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -110,7 +110,7 @@ * @opensearch.internal */ public class Lucene { - public static final String LATEST_CODEC = "Lucene99"; + public static final String LATEST_CODEC = "Lucene912"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; @@ -272,7 +272,7 @@ public static void checkSegmentInfoIntegrity(final Directory directory) throws I @Override protected Object doBody(String segmentFileName) throws IOException { - try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) { + try (IndexInput input = directory.openInput(segmentFileName, IOContext.READONCE)) { CodecUtil.checksumEntireFile(input); } return null; diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 59fafdf1ba74e..3a93795ef61ec 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; @@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe final MapBuilder codecs = MapBuilder.newMapBuilder(); assert null != indexSettings; if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene99Codec()); - codecs.put(LZ4, new Lucene99Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene912Codec()); + codecs.put(LZ4, new Lucene912Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION)); + codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION)); } else { // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec // We can still support all the compression codecs when composite index is present diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index 1ad17f121560c..0ed8c3880f2ea 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -37,7 +37,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat; import org.opensearch.index.codec.fuzzy.FuzzySetFactory; @@ -59,7 +59,7 @@ * * @opensearch.internal */ -public class PerFieldMappingPostingFormatCodec extends Lucene99Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene912Codec { private final Logger logger; private final MapperService mapperService; private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java index 99691d7061ac9..f5003417c1298 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java @@ -10,8 +10,9 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.index.codec.composite.composite99.Composite99Codec; import org.opensearch.index.mapper.MapperService; @@ -38,10 +39,10 @@ public CompositeCodecFactory() {} public Map getCompositeIndexCodecs(MapperService mapperService, Logger logger) { Map codecs = new HashMap<>(); - codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(DEFAULT_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + codecs.put(LZ4, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + codecs.put(BEST_COMPRESSION_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(ZLIB, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger)); return codecs; } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java new file mode 100644 index 0000000000000..9bbff7d344c2f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite.composite912; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.mapper.MapperService; + +/** + * Extends the Codec to support new file formats for composite indices eg: star tree index + * based on the mappings. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Composite912Codec extends FilterCodec { + public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite912Codec"; + private final MapperService mapperService; + + // needed for SPI - this is used in reader path + public Composite912Codec() { + this(COMPOSITE_INDEX_CODEC_NAME, new Lucene912Codec(), null); + } + + public Composite912Codec(Lucene912Codec.Mode compressionMode, MapperService mapperService, Logger logger) { + this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to + * this ctor. + * + * @param name name of the codec + * @param delegate codec delegate + * @param mapperService mapper service instance + */ + protected Composite912Codec(String name, Codec delegate, MapperService mapperService) { + super(name, delegate); + this.mapperService = mapperService; + } + + @Override + public DocValuesFormat docValuesFormat() { + return new Composite99DocValuesFormat(mapperService); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/package-info.java new file mode 100644 index 0000000000000..84520ed546e60 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Responsible for handling all composite index codecs and operations associated with Composite912 codec + */ +package org.opensearch.index.codec.composite.composite912; diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java index 8422932e937c2..20f1c7e9b7614 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java @@ -8,13 +8,11 @@ package org.opensearch.index.codec.composite.composite99; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; /** @@ -33,10 +31,6 @@ public Composite99Codec() { this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null); } - public Composite99Codec(Lucene99Codec.Mode compressionMode, MapperService mapperService, Logger logger) { - this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService); - } - /** * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to * this ctor. diff --git a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java index a9d9f6cb35fcb..fe48adf3249a3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/CompletionFieldMapper.java @@ -35,7 +35,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; -import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -330,7 +330,7 @@ public ContextMappings getContextMappings() { */ public static synchronized PostingsFormat postingsFormat() { if (postingsFormat == null) { - postingsFormat = new Completion99PostingsFormat(); + postingsFormat = new Completion912PostingsFormat(); } return postingsFormat; } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 6e12e4ed3da1a..eb3999718ca5b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -5214,7 +5214,7 @@ public void syncSegmentsFromGivenRemoteSegmentStore( } else if (segmentsNFile != null) { try ( ChecksumIndexInput indexInput = new BufferedChecksumIndexInput( - storeDirectory.openInput(segmentsNFile, IOContext.DEFAULT) + storeDirectory.openInput(segmentsNFile, IOContext.READONCE) ) ) { long commitGeneration = SegmentInfos.generationFromSegmentsFileName(segmentsNFile); @@ -5288,7 +5288,7 @@ private String copySegmentFiles( // Visible for testing boolean localDirectoryContains(Directory localDirectory, String file, long checksum) throws IOException { - try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { + try (IndexInput indexInput = localDirectory.openInput(file, IOContext.READONCE)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { return true; } else { diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 3db5e0b7f4b0c..702928771f131 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -459,7 +459,7 @@ private void uploadNewSegments( batchUploadListener.onFailure(ex); }); statsListener.beforeUpload(src); - remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload()); + remoteDirectory.copyFrom(storeDirectory, src, IOContext.READONCE, aggregatedListener, isLowPriorityUpload()); } } @@ -488,7 +488,7 @@ private boolean skipUpload(String file) { private String getChecksumOfLocalFile(String file) throws IOException { if (!localSegmentChecksumMap.containsKey(file)) { - try (IndexInput indexInput = storeDirectory.openInput(file, IOContext.DEFAULT)) { + try (IndexInput indexInput = storeDirectory.openInput(file, IOContext.READONCE)) { String checksum = Long.toString(CodecUtil.retrieveChecksum(indexInput)); localSegmentChecksumMap.put(file, checksum); } diff --git a/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java b/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java index 139b8fffbac3a..b485b827ca3af 100644 --- a/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java +++ b/server/src/main/java/org/opensearch/index/similarity/SimilarityProviders.java @@ -291,8 +291,7 @@ public static ClassicSimilarity createClassicSimilarity(Settings settings, Versi boolean discountOverlaps = settings.getAsBoolean(DISCOUNT_OVERLAPS, true); - ClassicSimilarity similarity = new ClassicSimilarity(); - similarity.setDiscountOverlaps(discountOverlaps); + ClassicSimilarity similarity = new ClassicSimilarity(discountOverlaps); return similarity; } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index c61fae74c0bc0..aa856aa22313d 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -424,7 +424,7 @@ private void uploadBlob( } private long calculateChecksumOfChecksum(Directory directory, String file) throws IOException { - try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) { + try (IndexInput indexInput = directory.openInput(file, IOContext.READONCE)) { try { return checksumOfChecksum(indexInput, SEGMENT_CHECKSUM_BYTES); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 5be516166803e..25d6e24cade14 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -759,7 +759,7 @@ private void tryAndDeleteLocalFile(String filename, Directory directory) { } private String getChecksumOfLocalFile(Directory directory, String file) throws IOException { - try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) { + try (IndexInput indexInput = directory.openInput(file, IOContext.READONCE)) { return Long.toString(CodecUtil.retrieveChecksum(indexInput)); } } diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 56fc5b1ffa90d..e7de61ff764f6 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -171,12 +171,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref Property.IndexScope ); - /** - * Specific {@link IOContext} used to verify Lucene files footer checksums. - * See {@link MetadataSnapshot#checksumFromLuceneFile(Directory, String, Map, Logger, Version, boolean)} - */ - public static final IOContext READONCE_CHECKSUM = new IOContext(IOContext.READONCE.context); - private final AtomicBoolean isClosed = new AtomicBoolean(false); private final StoreDirectory directory; private final ReentrantReadWriteLock metadataLock = new ReentrantReadWriteLock(); @@ -1209,7 +1203,7 @@ private static void checksumFromLuceneFile( ) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); - try (IndexInput in = directory.openInput(file, READONCE_CHECKSUM)) { + try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java index a70700b64eb6e..80ba5146535aa 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentFileTransferHandler.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; @@ -97,20 +98,26 @@ public MultiChunkTransfer createTransfer( return new MultiChunkTransfer<>(logger, threadPool.getThreadContext(), listener, maxConcurrentFileChunks, Arrays.asList(files)) { final Deque buffers = new ConcurrentLinkedDeque<>(); - InputStreamIndexInput currentInput = null; + volatile InputStreamIndexInput currentInput = null; long offset = 0; @Override protected void onNewResource(StoreFileMetadata md) throws IOException { offset = 0; IOUtils.close(currentInput, () -> currentInput = null); - final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE); - currentInput = new InputStreamIndexInput(indexInput, md.length()) { - @Override - public void close() throws IOException { - IOUtils.close(indexInput, super::close); // InputStreamIndexInput's close is a noop - } - }; + // Open all files other than Segments* using IOContext.READ. + // With Lucene9_12 a READONCE context will confine the underlying IndexInput (MemorySegmentIndexInput) to a single thread. + // Segments* files require IOContext.READONCE + // https://github.com/apache/lucene/blob/b2d3a2b37e00f19a74949097736be8fd64745f61/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java#L817 + if (md.name().startsWith(IndexFileNames.SEGMENTS) == false) { + final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READ); + currentInput = new InputStreamIndexInput(indexInput, md.length()) { + @Override + public void close() throws IOException { + IOUtils.close(indexInput, super::close); // InputStreamIndexInput's close is a noop + } + }; + } } private byte[] acquireBuffer() { @@ -126,7 +133,7 @@ protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException { assert Transports.assertNotTransportThread("read file chunk"); cancellableThreads.checkForCancel(); final byte[] buffer = acquireBuffer(); - final int bytesRead = currentInput.read(buffer); + final int bytesRead = readBytes(md, buffer); if (bytesRead == -1) { throw new CorruptIndexException("file truncated; length=" + md.length() + " offset=" + offset, md.name()); } @@ -142,6 +149,20 @@ protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException { return chunk; } + private int readBytes(StoreFileMetadata md, byte[] buffer) throws IOException { + // if we don't have a currentInput by now open once to create the chunk. + if (currentInput == null) { + try (IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) { + try (InputStreamIndexInput in = new InputStreamIndexInput(indexInput, md.length())) { + in.skip(offset); + return in.read(buffer); + } + } + } else { + return currentInput.read(buffer); + } + } + @Override protected void executeChunkRequest(FileChunk request, ActionListener listener1) { cancellableThreads.checkForCancel(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index af764556b7549..7131b49a41834 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -244,7 +244,7 @@ private List getFiles(CheckpointInfoResponse checkpointInfo) // pkg private for tests private boolean validateLocalChecksum(StoreFileMetadata file) { - try (IndexInput indexInput = indexShard.store().directory().openInput(file.name(), IOContext.DEFAULT)) { + try (IndexInput indexInput = indexShard.store().directory().openInput(file.name(), IOContext.READONCE)) { String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput)); if (file.checksum().equals(checksum)) { return true; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 14c201e819994..8482146530860 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -4530,7 +4530,7 @@ private void snapshotFile( ) throws IOException { final BlobContainer shardContainer = shardContainer(indexId, shardId); final String file = fileInfo.physicalName(); - try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) { + try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READ, fileInfo.metadata())) { for (int i = 0; i < fileInfo.numberOfParts(); i++) { final long partBytes = fileInfo.partBytes(i); diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index f118e4106db83..aa8212e8dad69 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -303,7 +303,8 @@ protected void search(List leaves, Weight weight, Collector c * {@link LeafCollector#collect(int)} is called for every matching document in * the provided ctx. */ - private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { + @Override + protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { // Check if at all we need to call this leaf for collecting results. if (canMatch(ctx) == false) { diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index f51452c57f975..432d7e4906c08 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1 +1,2 @@ org.opensearch.index.codec.composite.composite99.Composite99Codec +org.opensearch.index.codec.composite.composite912.Composite912Codec diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index bbf98b5c32918..cdd2c6d92f788 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -48,7 +48,7 @@ import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; @@ -71,58 +71,58 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(false); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene99Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene912Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService(false).codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); } public void testDefaultWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); - assert codec instanceof Composite99Codec; + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); + assert codec instanceof Composite912Codec; } public void testBestCompression() throws Exception { Codec codec = createCodecService(false).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); } public void testBestCompressionWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); - assert codec instanceof Composite99Codec; + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); + assert codec instanceof Composite912Codec; } public void testLZ4() throws Exception { Codec codec = createCodecService(false).codec("lz4"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); assert codec instanceof PerFieldMappingPostingFormatCodec; } public void testLZ4WithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("lz4"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); - assert codec instanceof Composite99Codec; + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); + assert codec instanceof Composite912Codec; } public void testZlib() throws Exception { Codec codec = createCodecService(false).codec("zlib"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); assert codec instanceof PerFieldMappingPostingFormatCodec; } public void testZlibWithCompositeIndex() throws Exception { Codec codec = createCodecService(false, true).codec("zlib"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); - assert codec instanceof Composite99Codec; + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); + assert codec instanceof Composite912Codec; } public void testResolveDefaultCodecsWithCompositeIndex() throws Exception { CodecService codecService = createCodecService(false, true); - assertThat(codecService.codec("default"), instanceOf(Composite99Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Composite912Codec.class)); } public void testBestCompressionWithCompressionLevel() { @@ -157,12 +157,12 @@ public void testLuceneCodecsWithCompressionLevel() { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); } public void testExceptionCodecNull() { @@ -174,11 +174,11 @@ public void testExceptionIndexSettingsNull() { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene912Codec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java index 7fb8fe7f68f45..5abb01dd02e1b 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.codec.composite; import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentInfo; @@ -44,12 +44,12 @@ public void testGetDocValuesConsumerForCompositeCodec() throws IOException { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", randomInt(), false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java index 55d637dfb9cae..67a953d923b22 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentInfo; @@ -48,12 +48,12 @@ public void testGetDocValuesProducerForCompositeCodec99() throws IOException { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", randomInt(), false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java similarity index 97% rename from server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java rename to server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index 1c267c67e60ed..4fe0199f89f41 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite99/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -6,14 +6,14 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite99.datacube.startree; +package org.opensearch.index.codec.composite912.datacube.startree; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; @@ -41,7 +41,7 @@ import org.opensearch.index.MapperTestUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -107,7 +107,7 @@ protected Codec getCodec() { } catch (IOException e) { throw new RuntimeException(e); } - Codec codec = new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, testLogger); + Codec codec = new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, testLogger); return codec; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index b77200f173e71..95fd1579c2f63 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.EmptyDocValuesProducer; @@ -273,7 +273,7 @@ private SegmentReadState getReadState(int numDocs, List dimensionFields, numDocs, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), writeState.segmentInfo.getId(), new HashMap<>(), @@ -287,12 +287,12 @@ private SegmentWriteState getWriteState(int numDocs, byte[] id) { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", numDocs, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), id, new HashMap<>(), @@ -1453,7 +1453,7 @@ public void test_build_multipleStarTrees() throws IOException { 7, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), @@ -1724,7 +1724,7 @@ public void test_build_starTreeDataset() throws IOException { 7, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 8f750b09c75ee..705700aada2ff 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -108,12 +108,12 @@ public static void setup() throws IOException { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", 5, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java index 94fe148eab6b3..4ab21dbce059f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentInfo; @@ -60,12 +60,12 @@ public void setUp() throws Exception { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", 5, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index 2127eeb9636d3..c8636426449ad 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -8,7 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.fileformats.meta; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -73,12 +73,12 @@ public void setup() throws IOException { SegmentInfo segmentInfo = new SegmentInfo( directory, Version.LATEST, - Version.LUCENE_9_11_0, + Version.LUCENE_9_12_0, "test_segment", 6, false, false, - new Lucene99Codec(), + new Lucene912Codec(), new HashMap<>(), UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), new HashMap<>(), diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index 0c87c384e0749..6a2db930aff9b 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -32,14 +32,14 @@ package org.opensearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.opensearch.OpenSearchException; @@ -69,8 +69,8 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion99PostingsFormat(); - indexWriterConfig.setCodec(new Lucene99Codec() { + final PostingsFormat postingsFormat = new Completion912PostingsFormat(); + indexWriterConfig.setCodec(new Lucene912Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java index 21bf580712761..9f31fb84ce6af 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardCorruptionTests.java @@ -43,7 +43,7 @@ public void testLocalDirectoryContains() throws IOException { } boolean corrupted = randomBoolean(); long checksum = 0; - try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { + try (IndexInput indexInput = localDirectory.openInput(file, IOContext.READONCE)) { checksum = CodecUtil.retrieveChecksum(indexInput); } if (corrupted) { diff --git a/server/src/test/java/org/opensearch/index/store/StoreTests.java b/server/src/test/java/org/opensearch/index/store/StoreTests.java index ab30a4c1c435f..3777753cb2de8 100644 --- a/server/src/test/java/org/opensearch/index/store/StoreTests.java +++ b/server/src/test/java/org/opensearch/index/store/StoreTests.java @@ -392,7 +392,7 @@ public void testNewChecksums() throws IOException { metadata = store.getMetadata(); assertThat(metadata.asMap().isEmpty(), is(false)); for (StoreFileMetadata meta : metadata) { - try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) { + try (IndexInput input = store.directory().openInput(meta.name(), IOContext.READONCE)) { String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input)); assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(), equalTo(checksum)); assertThat(meta.writtenBy(), equalTo(Version.LATEST)); diff --git a/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java b/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java index 67522bb618cf1..8e551a51a2094 100644 --- a/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/CorruptionUtils.java @@ -96,7 +96,7 @@ public static void corruptFile(Random random, Path... files) throws IOException long checksumAfterCorruption; long actualChecksumAfterCorruption; - try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { + try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.READONCE)) { assertThat(input.getFilePointer(), is(0L)); input.seek(input.length() - CodecUtil.footerLength()); checksumAfterCorruption = input.getChecksum(); From e885aa93279342c9ab219ecf612d887ff8de8af6 Mon Sep 17 00:00:00 2001 From: expani1729 <110471048+expani@users.noreply.github.com> Date: Tue, 8 Oct 2024 00:31:55 +0530 Subject: [PATCH 193/357] Latency improvements to Multi Term Aggregations (#14993) * Avoid deep copy and other allocation improvements * Refactoring based on PR Comments and added JavaDocs * Added more comments * Added character for Triggering Jenkins build * Changes to cover collectZeroDocEntries method * Updated comment based on change in method's functionality * Added test to cover branches in collectZeroDocEntriesIfRequired * Rebased and resolved changelog conflict --------- Signed-off-by: expani --- CHANGELOG.md | 1 + .../bucket/terms/MultiTermsAggregator.java | 96 ++++++++++++------- .../terms/MultiTermsAggregatorTests.java | 70 +++++++++++--- 3 files changed, 116 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e3a515fb8008..c2d021273f271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) +- Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java index 59f48bd7fbaba..07edf487af670 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregator.java @@ -33,6 +33,7 @@ import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalOrder; import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.bucket.BucketsAggregator; import org.opensearch.search.aggregations.bucket.DeferableBucketAggregator; import org.opensearch.search.aggregations.bucket.LocalBucketCountThresholds; import org.opensearch.search.aggregations.support.AggregationPath; @@ -215,19 +216,11 @@ public InternalAggregation buildEmptyAggregation() { @Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { - MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx); + MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx, bucketOrds, this, sub); return new LeafBucketCollector() { @Override public void collect(int doc, long owningBucketOrd) throws IOException { - for (BytesRef compositeKey : collector.apply(doc)) { - long bucketOrd = bucketOrds.add(owningBucketOrd, compositeKey); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - } + collector.apply(doc, owningBucketOrd); } }; } @@ -268,12 +261,10 @@ private void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOExcept } // we need to fill-in the blanks for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) { - MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx); // brute force + MultiTermsValuesSourceCollector collector = multiTermsValue.getValues(ctx, bucketOrds, null, null); for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - for (BytesRef compositeKey : collector.apply(docId)) { - bucketOrds.add(owningBucketOrd, compositeKey); - } + collector.apply(docId, owningBucketOrd); } } } @@ -284,10 +275,11 @@ private void collectZeroDocEntriesIfNeeded(long owningBucketOrd) throws IOExcept @FunctionalInterface interface MultiTermsValuesSourceCollector { /** - * Collect a list values of multi_terms on each doc. - * Each terms could have multi_values, so the result is the cartesian product of each term's values. + * Generates the cartesian product of all fields used in aggregation and + * collects them in buckets using the composite key of their field values. */ - List apply(int doc) throws IOException; + void apply(int doc, long owningBucketOrd) throws IOException; + } @FunctionalInterface @@ -361,47 +353,72 @@ public MultiTermsValuesSource(List valuesSources) { this.valuesSources = valuesSources; } - public MultiTermsValuesSourceCollector getValues(LeafReaderContext ctx) throws IOException { + public MultiTermsValuesSourceCollector getValues( + LeafReaderContext ctx, + BytesKeyedBucketOrds bucketOrds, + BucketsAggregator aggregator, + LeafBucketCollector sub + ) throws IOException { List collectors = new ArrayList<>(); for (InternalValuesSource valuesSource : valuesSources) { collectors.add(valuesSource.apply(ctx)); } + boolean collectBucketOrds = aggregator != null && sub != null; return new MultiTermsValuesSourceCollector() { + + /** + * This method does the following :
+ *
  • Fetches the values of every field present in the doc List>> via @{@link InternalValuesSourceCollector}
    • + *
  • Generates Composite keys from the fetched values for all fields present in the aggregation.
    • + *
  • Adds every composite key to the @{@link BytesKeyedBucketOrds} and Optionally collects them via @{@link BucketsAggregator#collectBucket(LeafBucketCollector, int, long)}
    • + */ @Override - public List apply(int doc) throws IOException { + public void apply(int doc, long owningBucketOrd) throws IOException { + // TODO A new list creation can be avoided for every doc. List>> collectedValues = new ArrayList<>(); for (InternalValuesSourceCollector collector : collectors) { collectedValues.add(collector.apply(doc)); } - List result = new ArrayList<>(); scratch.seek(0); scratch.writeVInt(collectors.size()); // number of fields per composite key - cartesianProduct(result, scratch, collectedValues, 0); - return result; + generateAndCollectCompositeKeys(collectedValues, 0, owningBucketOrd, doc); } /** - * Cartesian product using depth first search. - * - *

    - * Composite keys are encoded to a {@link BytesRef} in a format compatible with {@link StreamOutput::writeGenericValue}, - * but reuses the encoding of the shared prefixes from the previous levels to avoid wasteful work. + * This generates and collects all Composite keys in their buckets by performing a cartesian product
    + * of all the values in all the fields ( used in agg ) for the given doc recursively. + * @param collectedValues : Values of all fields present in the aggregation for the @doc + * @param index : Points to the field being added to generate the composite key */ - private void cartesianProduct( - List compositeKeys, - BytesStreamOutput scratch, + private void generateAndCollectCompositeKeys( List>> collectedValues, - int index + int index, + long owningBucketOrd, + int doc ) throws IOException { if (collectedValues.size() == index) { - compositeKeys.add(BytesRef.deepCopyOf(scratch.bytes().toBytesRef())); + // Avoid performing a deep copy of the composite key by inlining. + long bucketOrd = bucketOrds.add(owningBucketOrd, scratch.bytes().toBytesRef()); + if (collectBucketOrds) { + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + aggregator.collectExistingBucket(sub, doc, bucketOrd); + } else { + aggregator.collectBucket(sub, doc, bucketOrd); + } + } return; } long position = scratch.position(); - for (TermValue value : collectedValues.get(index)) { + List> values = collectedValues.get(index); + int numIterations = values.size(); + // For each loop is not done to reduce the allocations done for Iterator objects + // once for every field in every doc. + for (int i = 0; i < numIterations; i++) { + TermValue value = values.get(i); value.writeTo(scratch); // encode the value - cartesianProduct(compositeKeys, scratch, collectedValues, index + 1); // dfs + generateAndCollectCompositeKeys(collectedValues, index + 1, owningBucketOrd, doc); // dfs scratch.seek(position); // backtrack } } @@ -441,9 +458,14 @@ static InternalValuesSource bytesValuesSource(ValuesSource valuesSource, Include if (i > 0 && bytes.equals(previous)) { continue; } - BytesRef copy = BytesRef.deepCopyOf(bytes); - termValues.add(TermValue.of(copy)); - previous = copy; + // Performing a deep copy is not required for field containing only one value. + if (valuesCount > 1) { + BytesRef copy = BytesRef.deepCopyOf(bytes); + termValues.add(TermValue.of(copy)); + previous = copy; + } else { + termValues.add(TermValue.of(bytes)); + } } return termValues; }; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java index d550c4c354c0f..bb46c5607a4a7 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java @@ -126,6 +126,19 @@ public class MultiTermsAggregatorTests extends AggregatorTestCase { private static final Consumer NONE_DECORATOR = null; + private static final Consumer IP_AND_KEYWORD_DESC_ORDER_VERIFY = h -> { + MatcherAssert.assertThat(h.getBuckets(), hasSize(3)); + MatcherAssert.assertThat(h.getBuckets().get(0).getKey(), contains(equalTo("a"), equalTo("192.168.0.0"))); + MatcherAssert.assertThat(h.getBuckets().get(0).getKeyAsString(), equalTo("a|192.168.0.0")); + MatcherAssert.assertThat(h.getBuckets().get(0).getDocCount(), equalTo(2L)); + MatcherAssert.assertThat(h.getBuckets().get(1).getKey(), contains(equalTo("b"), equalTo("192.168.0.1"))); + MatcherAssert.assertThat(h.getBuckets().get(1).getKeyAsString(), equalTo("b|192.168.0.1")); + MatcherAssert.assertThat(h.getBuckets().get(1).getDocCount(), equalTo(1L)); + MatcherAssert.assertThat(h.getBuckets().get(2).getKey(), contains(equalTo("c"), equalTo("192.168.0.2"))); + MatcherAssert.assertThat(h.getBuckets().get(2).getKeyAsString(), equalTo("c|192.168.0.2")); + MatcherAssert.assertThat(h.getBuckets().get(2).getDocCount(), equalTo(1L)); + }; + @Override protected List getSupportedValuesSourceTypes() { return Collections.unmodifiableList( @@ -672,8 +685,48 @@ public void testDatesFieldFormat() throws IOException { ); } - public void testIpAndKeyword() throws IOException { - testAggregation(new MatchAllDocsQuery(), fieldConfigs(asList(KEYWORD_FIELD, IP_FIELD)), NONE_DECORATOR, iw -> { + public void testIpAndKeywordDefaultDescOrder() throws IOException { + ipAndKeywordTest(NONE_DECORATOR, IP_AND_KEYWORD_DESC_ORDER_VERIFY); + } + + public void testIpAndKeywordWithBucketCountSameAsSize() throws IOException { + ipAndKeywordTest(multiTermsAggregationBuilder -> { + multiTermsAggregationBuilder.minDocCount(0); + multiTermsAggregationBuilder.size(3); + multiTermsAggregationBuilder.order(BucketOrder.compound(BucketOrder.count(false))); + }, IP_AND_KEYWORD_DESC_ORDER_VERIFY); + } + + public void testIpAndKeywordWithBucketCountGreaterThanSize() throws IOException { + ipAndKeywordTest(multiTermsAggregationBuilder -> { + multiTermsAggregationBuilder.minDocCount(0); + multiTermsAggregationBuilder.size(10); + multiTermsAggregationBuilder.order(BucketOrder.compound(BucketOrder.count(false))); + }, IP_AND_KEYWORD_DESC_ORDER_VERIFY); + } + + public void testIpAndKeywordAscOrder() throws IOException { + ipAndKeywordTest(multiTermsAggregationBuilder -> { + multiTermsAggregationBuilder.minDocCount(0); + multiTermsAggregationBuilder.size(3); + multiTermsAggregationBuilder.order(BucketOrder.compound(BucketOrder.count(true))); + }, h -> { + MatcherAssert.assertThat(h.getBuckets(), hasSize(3)); + MatcherAssert.assertThat(h.getBuckets().get(0).getKey(), contains(equalTo("b"), equalTo("192.168.0.1"))); + MatcherAssert.assertThat(h.getBuckets().get(0).getKeyAsString(), equalTo("b|192.168.0.1")); + MatcherAssert.assertThat(h.getBuckets().get(0).getDocCount(), equalTo(1L)); + MatcherAssert.assertThat(h.getBuckets().get(1).getKey(), contains(equalTo("c"), equalTo("192.168.0.2"))); + MatcherAssert.assertThat(h.getBuckets().get(1).getKeyAsString(), equalTo("c|192.168.0.2")); + MatcherAssert.assertThat(h.getBuckets().get(1).getDocCount(), equalTo(1L)); + MatcherAssert.assertThat(h.getBuckets().get(2).getKey(), contains(equalTo("a"), equalTo("192.168.0.0"))); + MatcherAssert.assertThat(h.getBuckets().get(2).getKeyAsString(), equalTo("a|192.168.0.0")); + MatcherAssert.assertThat(h.getBuckets().get(2).getDocCount(), equalTo(2L)); + }); + } + + private void ipAndKeywordTest(Consumer builderDecorator, Consumer verify) + throws IOException { + testAggregation(new MatchAllDocsQuery(), fieldConfigs(asList(KEYWORD_FIELD, IP_FIELD)), builderDecorator, iw -> { iw.addDocument( asList( new SortedDocValuesField(KEYWORD_FIELD, new BytesRef("a")), @@ -698,18 +751,7 @@ public void testIpAndKeyword() throws IOException { new SortedDocValuesField(IP_FIELD, new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.0")))) ) ); - }, h -> { - MatcherAssert.assertThat(h.getBuckets(), hasSize(3)); - MatcherAssert.assertThat(h.getBuckets().get(0).getKey(), contains(equalTo("a"), equalTo("192.168.0.0"))); - MatcherAssert.assertThat(h.getBuckets().get(0).getKeyAsString(), equalTo("a|192.168.0.0")); - MatcherAssert.assertThat(h.getBuckets().get(0).getDocCount(), equalTo(2L)); - MatcherAssert.assertThat(h.getBuckets().get(1).getKey(), contains(equalTo("b"), equalTo("192.168.0.1"))); - MatcherAssert.assertThat(h.getBuckets().get(1).getKeyAsString(), equalTo("b|192.168.0.1")); - MatcherAssert.assertThat(h.getBuckets().get(1).getDocCount(), equalTo(1L)); - MatcherAssert.assertThat(h.getBuckets().get(2).getKey(), contains(equalTo("c"), equalTo("192.168.0.2"))); - MatcherAssert.assertThat(h.getBuckets().get(2).getKeyAsString(), equalTo("c|192.168.0.2")); - MatcherAssert.assertThat(h.getBuckets().get(2).getDocCount(), equalTo(1L)); - }); + }, verify); } public void testEmpty() throws IOException { From e24b4c98405d95b781a43fe73ca370c1a1ca452a Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Mon, 7 Oct 2024 17:21:20 -0700 Subject: [PATCH 194/357] Add wlm resiliency orchestrator (query group service) (#15925) * cancellation related Signed-off-by: Kiran Prakash * Update CHANGELOG.md Signed-off-by: Kiran Prakash * add better cancellation reason Signed-off-by: Kiran Prakash * Update DefaultTaskCancellationTests.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskCancellation.java Signed-off-by: Kiran Prakash * Update DefaultTaskSelectionStrategy.java Signed-off-by: Kiran Prakash * refactor Signed-off-by: Kiran Prakash * refactor node level threshold Signed-off-by: Kiran Prakash * use query group task Signed-off-by: Kaushal Kumar * code clean up and refactorings Signed-off-by: Kaushal Kumar * add unit tests and fix existing ones Signed-off-by: Kaushal Kumar * uncomment the test case Signed-off-by: Kaushal Kumar * update CHANGELOG Signed-off-by: Kaushal Kumar * fix imports Signed-off-by: Kaushal Kumar * add queryGroupService Signed-off-by: Kaushal Kumar * refactor and add UTs for new constructs Signed-off-by: Kaushal Kumar * fix javadocs Signed-off-by: Kaushal Kumar * remove code clutter Signed-off-by: Kaushal Kumar * change annotation version and task selection strategy Signed-off-by: Kaushal Kumar * rename a util class Signed-off-by: Kaushal Kumar * remove wrappers from resource type Signed-off-by: Kaushal Kumar * apply spotless Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * add rename changes Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * initial changes Signed-off-by: Kaushal Kumar * refactor changes and logical bug fix Signed-off-by: Kaushal Kumar * add chanegs Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * temp changes Signed-off-by: Kaushal Kumar * add UTs Signed-off-by: Kaushal Kumar * add changelog Signed-off-by: Kaushal Kumar * add task completion listener hook Signed-off-by: Kaushal Kumar * add remaining pieces to make the feature functional Signed-off-by: Kaushal Kumar * extend stats and fix bugs Signed-off-by: Kaushal Kumar * fix bugs and add logic to make SBP work with wlm Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * fix bugs and SBP ITs Signed-off-by: Kaushal Kumar * add missed applyCluster state change Signed-off-by: Kaushal Kumar * address comments Signed-off-by: Kaushal Kumar * decouple queryGroupService and cancellationService Signed-off-by: Kaushal Kumar * replace StateApplier with StateListener interface Signed-off-by: Kaushal Kumar * fix precommit errors Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kiran Prakash Signed-off-by: Kaushal Kumar Co-authored-by: Kiran Prakash --- CHANGELOG.md | 1 + .../backpressure/SearchBackpressureIT.java | 2 + .../org/opensearch/action/ActionModule.java | 6 +- .../common/settings/ClusterSettings.java | 3 + .../main/java/org/opensearch/node/Node.java | 39 +- .../SearchBackpressureService.java | 13 +- .../org/opensearch/wlm/QueryGroupService.java | 290 ++++++++++- .../org/opensearch/wlm/QueryGroupTask.java | 9 +- .../wlm/QueryGroupsStateAccessor.java | 64 +++ .../main/java/org/opensearch/wlm/WlmMode.java | 40 ++ .../wlm/WorkloadManagementSettings.java | 105 ++++ .../QueryGroupTaskCancellationService.java | 123 ++++- .../opensearch/wlm/stats/QueryGroupState.java | 29 +- .../opensearch/wlm/stats/QueryGroupStats.java | 11 +- ...QueryGroupResourceUsageTrackerService.java | 2 + .../SearchBackpressureServiceTests.java | 52 +- .../wlm/QueryGroupServiceTests.java | 489 ++++++++++++++++++ ...adManagementTransportInterceptorTests.java | 36 +- ...ueryGroupTaskCancellationServiceTests.java | 100 +++- ...eryGroupRequestOperationListenerTests.java | 97 +++- .../wlm/stats/QueryGroupStateTests.java | 10 +- .../wlm/stats/QueryGroupStatsTests.java | 6 +- 22 files changed, 1410 insertions(+), 117 deletions(-) create mode 100644 server/src/main/java/org/opensearch/wlm/QueryGroupsStateAccessor.java create mode 100644 server/src/main/java/org/opensearch/wlm/WlmMode.java create mode 100644 server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d021273f271..ab8cdd882bbb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) +- [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java index fb84134120e00..40c9301ef4bce 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/SearchBackpressureIT.java @@ -39,6 +39,7 @@ import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import org.opensearch.wlm.QueryGroupTask; import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; @@ -411,6 +412,7 @@ protected void doExecute(Task task, TestRequest request, ActionListener { try { CancellableTask cancellableTask = (CancellableTask) task; + ((QueryGroupTask) task).setQueryGroupId(threadPool.getThreadContext()); long startTime = System.nanoTime(); // Doing a busy-wait until task cancellation or timeout. diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index d6b4a93b34217..12c7626e385a4 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -481,6 +481,7 @@ import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.usage.UsageService; +import org.opensearch.wlm.QueryGroupTask; import java.util.ArrayList; import java.util.Collections; @@ -565,7 +566,10 @@ public ActionModule( destructiveOperations = new DestructiveOperations(settings, clusterSettings); Set headers = Stream.concat( actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()), - Stream.of(new RestHeaderDefinition(Task.X_OPAQUE_ID, false)) + Stream.of( + new RestHeaderDefinition(Task.X_OPAQUE_ID, false), + new RestHeaderDefinition(QueryGroupTask.QUERY_GROUP_ID_HEADER, false) + ) ).collect(Collectors.toSet()); UnaryOperator restWrapper = null; for (ActionPlugin plugin : actionPlugins) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index e18c4dabb29eb..a84a29256ee19 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -798,6 +798,9 @@ public void apply(Settings value, Settings current, Settings previous) { WorkloadManagementSettings.NODE_LEVEL_CPU_CANCELLATION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_MEMORY_REJECTION_THRESHOLD, WorkloadManagementSettings.NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD, + WorkloadManagementSettings.WLM_MODE_SETTING, + WorkloadManagementSettings.QUERYGROUP_SERVICE_RUN_INTERVAL_SETTING, + WorkloadManagementSettings.QUERYGROUP_SERVICE_DURESS_STREAK_SETTING, // Settings to be used for limiting rest requests ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING, diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index fe4bfa1e64ba1..ddba10369ff05 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -270,8 +270,13 @@ import org.opensearch.usage.UsageService; import org.opensearch.watcher.ResourceWatcherService; import org.opensearch.wlm.QueryGroupService; +import org.opensearch.wlm.QueryGroupsStateAccessor; +import org.opensearch.wlm.WorkloadManagementSettings; import org.opensearch.wlm.WorkloadManagementTransportInterceptor; +import org.opensearch.wlm.cancellation.MaximumResourceTaskSelectionStrategy; +import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService; import org.opensearch.wlm.listeners.QueryGroupRequestOperationListener; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; import javax.net.ssl.SNIHostName; @@ -1022,8 +1027,30 @@ protected Node( List identityAwarePlugins = pluginsService.filterPlugins(IdentityAwarePlugin.class); identityService.initializeIdentityAwarePlugins(identityAwarePlugins); - final QueryGroupService queryGroupService = new QueryGroupService(); // We will need to replace this with actual instance of the - // queryGroupService + final QueryGroupResourceUsageTrackerService queryGroupResourceUsageTrackerService = new QueryGroupResourceUsageTrackerService( + taskResourceTrackingService + ); + final WorkloadManagementSettings workloadManagementSettings = new WorkloadManagementSettings( + settings, + settingsModule.getClusterSettings() + ); + + final QueryGroupsStateAccessor queryGroupsStateAccessor = new QueryGroupsStateAccessor(); + + final QueryGroupService queryGroupService = new QueryGroupService( + new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + queryGroupResourceUsageTrackerService, + queryGroupsStateAccessor + ), + clusterService, + threadPool, + workloadManagementSettings, + queryGroupsStateAccessor + ); + taskResourceTrackingService.addTaskCompletionListener(queryGroupService); + final QueryGroupRequestOperationListener queryGroupRequestOperationListener = new QueryGroupRequestOperationListener( queryGroupService, threadPool @@ -1089,7 +1116,7 @@ protected Node( WorkloadManagementTransportInterceptor workloadManagementTransportInterceptor = new WorkloadManagementTransportInterceptor( threadPool, - new QueryGroupService() // We will need to replace this with actual implementation + queryGroupService ); final Collection secureSettingsFactories = pluginsService.filterPlugins(Plugin.class) @@ -1184,7 +1211,8 @@ protected Node( searchBackpressureSettings, taskResourceTrackingService, threadPool, - transportService.getTaskManager() + transportService.getTaskManager(), + queryGroupService ); final SegmentReplicationStatsTracker segmentReplicationStatsTracker = new SegmentReplicationStatsTracker(indicesService); @@ -1396,6 +1424,7 @@ protected Node( b.bind(IndexingPressureService.class).toInstance(indexingPressureService); b.bind(TaskResourceTrackingService.class).toInstance(taskResourceTrackingService); b.bind(SearchBackpressureService.class).toInstance(searchBackpressureService); + b.bind(QueryGroupService.class).toInstance(queryGroupService); b.bind(AdmissionControlService.class).toInstance(admissionControlService); b.bind(UsageService.class).toInstance(usageService); b.bind(AggregationUsageService.class).toInstance(searchModule.getValuesSourceRegistry().getUsageService()); @@ -1589,6 +1618,7 @@ public Node start() throws NodeValidationException { nodeService.getMonitorService().start(); nodeService.getSearchBackpressureService().start(); nodeService.getTaskCancellationMonitoringService().start(); + injector.getInstance(QueryGroupService.class).start(); final ClusterService clusterService = injector.getInstance(ClusterService.class); @@ -1762,6 +1792,7 @@ private Node stop() { injector.getInstance(FsHealthService.class).stop(); injector.getInstance(NodeResourceUsageTracker.class).stop(); injector.getInstance(ResourceUsageCollectorService.class).stop(); + injector.getInstance(QueryGroupService.class).stop(); nodeService.getMonitorService().stop(); nodeService.getSearchBackpressureService().stop(); injector.getInstance(GatewayService.class).stop(); diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index e98046ba1dede..d6eac45ca0288 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -42,6 +42,7 @@ import org.opensearch.tasks.TaskResourceTrackingService.TaskCompletionListener; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.QueryGroupService; import org.opensearch.wlm.ResourceType; import java.io.IOException; @@ -86,12 +87,14 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem private final Map, SearchBackpressureState> searchBackpressureStates; private final TaskManager taskManager; + private final QueryGroupService queryGroupService; public SearchBackpressureService( SearchBackpressureSettings settings, TaskResourceTrackingService taskResourceTrackingService, ThreadPool threadPool, - TaskManager taskManager + TaskManager taskManager, + QueryGroupService queryGroupService ) { this(settings, taskResourceTrackingService, threadPool, System::nanoTime, new NodeDuressTrackers(new EnumMap<>(ResourceType.class) { { @@ -131,7 +134,8 @@ public SearchBackpressureService( settings.getClusterSettings(), SearchShardTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE ), - taskManager + taskManager, + queryGroupService ); } @@ -143,7 +147,8 @@ public SearchBackpressureService( NodeDuressTrackers nodeDuressTrackers, TaskResourceUsageTrackers searchTaskTrackers, TaskResourceUsageTrackers searchShardTaskTrackers, - TaskManager taskManager + TaskManager taskManager, + QueryGroupService queryGroupService ) { this.settings = settings; this.taskResourceTrackingService = taskResourceTrackingService; @@ -151,6 +156,7 @@ public SearchBackpressureService( this.threadPool = threadPool; this.nodeDuressTrackers = nodeDuressTrackers; this.taskManager = taskManager; + this.queryGroupService = queryGroupService; this.searchBackpressureStates = Map.of( SearchTask.class, @@ -346,6 +352,7 @@ List getTa .stream() .filter(type::isInstance) .map(type::cast) + .filter(queryGroupService::shouldSBPHandle) .collect(Collectors.toUnmodifiableList()); } diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java index 6545598dd9951..cda5916db26f3 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java @@ -8,36 +8,192 @@ package org.opensearch.wlm; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchShardTask; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.monitor.jvm.JvmStats; +import org.opensearch.monitor.process.ProcessProbe; +import org.opensearch.search.backpressure.trackers.NodeDuressTrackers; +import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.threadpool.Scheduler; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService; import org.opensearch.wlm.stats.QueryGroupState; import org.opensearch.wlm.stats.QueryGroupStats; import org.opensearch.wlm.stats.QueryGroupStats.QueryGroupStatsHolder; +import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService.TRACKED_RESOURCES; /** * As of now this is a stub and main implementation PR will be raised soon.Coming PR will collate these changes with core QueryGroupService changes */ -public class QueryGroupService { - // This map does not need to be concurrent since we will process the cluster state change serially and update - // this map with new additions and deletions of entries. QueryGroupState is thread safe - private final Map queryGroupStateMap; +public class QueryGroupService extends AbstractLifecycleComponent + implements + ClusterStateListener, + TaskResourceTrackingService.TaskCompletionListener { + + private static final Logger logger = LogManager.getLogger(QueryGroupService.class); + + private final QueryGroupTaskCancellationService taskCancellationService; + private volatile Scheduler.Cancellable scheduledFuture; + private final ThreadPool threadPool; + private final ClusterService clusterService; + private final WorkloadManagementSettings workloadManagementSettings; + private Set activeQueryGroups; + private final Set deletedQueryGroups; + private final NodeDuressTrackers nodeDuressTrackers; + private final QueryGroupsStateAccessor queryGroupsStateAccessor; + + public QueryGroupService( + QueryGroupTaskCancellationService taskCancellationService, + ClusterService clusterService, + ThreadPool threadPool, + WorkloadManagementSettings workloadManagementSettings, + QueryGroupsStateAccessor queryGroupsStateAccessor + ) { + + this( + taskCancellationService, + clusterService, + threadPool, + workloadManagementSettings, + new NodeDuressTrackers( + Map.of( + ResourceType.CPU, + new NodeDuressTracker( + () -> workloadManagementSettings.getNodeLevelCpuCancellationThreshold() < ProcessProbe.getInstance() + .getProcessCpuPercent() / 100.0, + workloadManagementSettings::getDuressStreak + ), + ResourceType.MEMORY, + new NodeDuressTracker( + () -> workloadManagementSettings.getNodeLevelMemoryCancellationThreshold() <= JvmStats.jvmStats() + .getMem() + .getHeapUsedPercent() / 100.0, + workloadManagementSettings::getDuressStreak + ) + ) + ), + queryGroupsStateAccessor, + new HashSet<>(), + new HashSet<>() + ); + } + + public QueryGroupService( + QueryGroupTaskCancellationService taskCancellationService, + ClusterService clusterService, + ThreadPool threadPool, + WorkloadManagementSettings workloadManagementSettings, + NodeDuressTrackers nodeDuressTrackers, + QueryGroupsStateAccessor queryGroupsStateAccessor, + Set activeQueryGroups, + Set deletedQueryGroups + ) { + this.taskCancellationService = taskCancellationService; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.workloadManagementSettings = workloadManagementSettings; + this.nodeDuressTrackers = nodeDuressTrackers; + this.activeQueryGroups = activeQueryGroups; + this.deletedQueryGroups = deletedQueryGroups; + this.queryGroupsStateAccessor = queryGroupsStateAccessor; + activeQueryGroups.forEach(queryGroup -> this.queryGroupsStateAccessor.addNewQueryGroup(queryGroup.get_id())); + this.queryGroupsStateAccessor.addNewQueryGroup(QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); + this.clusterService.addListener(this); + } + + /** + * run at regular interval + */ + void doRun() { + if (workloadManagementSettings.getWlmMode() == WlmMode.DISABLED) { + return; + } + taskCancellationService.cancelTasks(nodeDuressTrackers::isNodeInDuress, activeQueryGroups, deletedQueryGroups); + taskCancellationService.pruneDeletedQueryGroups(deletedQueryGroups); + } + + /** + * {@link AbstractLifecycleComponent} lifecycle method + */ + @Override + protected void doStart() { + scheduledFuture = threadPool.scheduleWithFixedDelay(() -> { + try { + doRun(); + } catch (Exception e) { + logger.debug("Exception occurred in Query Sandbox service", e); + } + }, this.workloadManagementSettings.getQueryGroupServiceRunInterval(), ThreadPool.Names.GENERIC); + } - public QueryGroupService() { - this(new HashMap<>()); + @Override + protected void doStop() { + if (scheduledFuture != null) { + scheduledFuture.cancel(); + } } - public QueryGroupService(Map queryGroupStateMap) { - this.queryGroupStateMap = queryGroupStateMap; + @Override + protected void doClose() throws IOException {} + + @Override + public void clusterChanged(ClusterChangedEvent event) { + // Retrieve the current and previous cluster states + Metadata previousMetadata = event.previousState().metadata(); + Metadata currentMetadata = event.state().metadata(); + + // Extract the query groups from both the current and previous cluster states + Map previousQueryGroups = previousMetadata.queryGroups(); + Map currentQueryGroups = currentMetadata.queryGroups(); + + // Detect new query groups added in the current cluster state + for (String queryGroupName : currentQueryGroups.keySet()) { + if (!previousQueryGroups.containsKey(queryGroupName)) { + // New query group detected + QueryGroup newQueryGroup = currentQueryGroups.get(queryGroupName); + // Perform any necessary actions with the new query group + queryGroupsStateAccessor.addNewQueryGroup(newQueryGroup.get_id()); + } + } + + // Detect query groups deleted in the current cluster state + for (String queryGroupName : previousQueryGroups.keySet()) { + if (!currentQueryGroups.containsKey(queryGroupName)) { + // Query group deleted + QueryGroup deletedQueryGroup = previousQueryGroups.get(queryGroupName); + // Perform any necessary actions with the deleted query group + this.deletedQueryGroups.add(deletedQueryGroup); + queryGroupsStateAccessor.removeQueryGroup(deletedQueryGroup.get_id()); + } + } + this.activeQueryGroups = new HashSet<>(currentMetadata.queryGroups().values()); } /** * updates the failure stats for the query group + * * @param queryGroupId query group identifier */ public void incrementFailuresFor(final String queryGroupId) { - QueryGroupState queryGroupState = queryGroupStateMap.get(queryGroupId); + QueryGroupState queryGroupState = queryGroupsStateAccessor.getQueryGroupState(queryGroupId); // This can happen if the request failed for a deleted query group // or new queryGroup is being created and has not been acknowledged yet if (queryGroupState == null) { @@ -47,12 +203,11 @@ public void incrementFailuresFor(final String queryGroupId) { } /** - * * @return node level query group stats */ public QueryGroupStats nodeStats() { final Map statsHolderMap = new HashMap<>(); - for (Map.Entry queryGroupsState : queryGroupStateMap.entrySet()) { + for (Map.Entry queryGroupsState : queryGroupsStateAccessor.getQueryGroupStateMap().entrySet()) { final String queryGroupId = queryGroupsState.getKey(); final QueryGroupState currentState = queryGroupsState.getValue(); @@ -63,18 +218,113 @@ public QueryGroupStats nodeStats() { } /** - * * @param queryGroupId query group identifier */ public void rejectIfNeeded(String queryGroupId) { - if (queryGroupId == null) return; - boolean reject = false; - final StringBuilder reason = new StringBuilder(); - // TODO: At this point this is dummy and we need to decide whether to cancel the request based on last - // reported resource usage for the queryGroup. We also need to increment the rejection count here for the - // query group - if (reject) { - throw new OpenSearchRejectedExecutionException("QueryGroup " + queryGroupId + " is already contended." + reason.toString()); + if (workloadManagementSettings.getWlmMode() != WlmMode.ENABLED) { + return; + } + + if (queryGroupId == null || queryGroupId.equals(QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get())) return; + QueryGroupState queryGroupState = queryGroupsStateAccessor.getQueryGroupState(queryGroupId); + + // This can happen if the request failed for a deleted query group + // or new queryGroup is being created and has not been acknowledged yet or invalid query group id + if (queryGroupState == null) { + return; + } + + // rejections will not happen for SOFT mode QueryGroups + Optional optionalQueryGroup = activeQueryGroups.stream().filter(x -> x.get_id().equals(queryGroupId)).findFirst(); + + if (optionalQueryGroup.isPresent() && optionalQueryGroup.get().getResiliencyMode() == MutableQueryGroupFragment.ResiliencyMode.SOFT) + return; + + optionalQueryGroup.ifPresent(queryGroup -> { + boolean reject = false; + final StringBuilder reason = new StringBuilder(); + for (ResourceType resourceType : TRACKED_RESOURCES) { + if (queryGroup.getResourceLimits().containsKey(resourceType)) { + final double threshold = getNormalisedRejectionThreshold( + queryGroup.getResourceLimits().get(resourceType), + resourceType + ); + final double lastRecordedUsage = queryGroupState.getResourceState().get(resourceType).getLastRecordedUsage(); + if (threshold < lastRecordedUsage) { + reject = true; + reason.append(resourceType) + .append(" limit is breaching for ENFORCED type QueryGroup: (") + .append(threshold) + .append(" < ") + .append(lastRecordedUsage) + .append("). "); + queryGroupState.getResourceState().get(resourceType).rejections.inc(); + // should not double count even if both the resource limits are breaching + break; + } + } + } + if (reject) { + queryGroupState.totalRejections.inc(); + throw new OpenSearchRejectedExecutionException( + "QueryGroup " + queryGroupId + " is already contended. " + reason.toString() + ); + } + }); + } + + private double getNormalisedRejectionThreshold(double limit, ResourceType resourceType) { + if (resourceType == ResourceType.CPU) { + return limit * workloadManagementSettings.getNodeLevelCpuRejectionThreshold(); + } else if (resourceType == ResourceType.MEMORY) { + return limit * workloadManagementSettings.getNodeLevelMemoryRejectionThreshold(); + } + throw new IllegalArgumentException(resourceType + " is not supported in WLM yet"); + } + + public Set getActiveQueryGroups() { + return activeQueryGroups; + } + + public Set getDeletedQueryGroups() { + return deletedQueryGroups; + } + + /** + * This method determines whether the task should be accounted by SBP if both features co-exist + * @param t QueryGroupTask + * @return whether or not SBP handle it + */ + public boolean shouldSBPHandle(Task t) { + QueryGroupTask task = (QueryGroupTask) t; + boolean isInvalidQueryGroupTask = true; + if (!task.getQueryGroupId().equals(QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get())) { + isInvalidQueryGroupTask = activeQueryGroups.stream() + .noneMatch(queryGroup -> queryGroup.get_id().equals(task.getQueryGroupId())); + } + return workloadManagementSettings.getWlmMode() != WlmMode.ENABLED || isInvalidQueryGroupTask; + } + + @Override + public void onTaskCompleted(Task task) { + if (!(task instanceof QueryGroupTask)) { + return; + } + final QueryGroupTask queryGroupTask = (QueryGroupTask) task; + String queryGroupId = queryGroupTask.getQueryGroupId(); + + // set the default queryGroupId if not existing in the active query groups + String finalQueryGroupId = queryGroupId; + boolean exists = activeQueryGroups.stream().anyMatch(queryGroup -> queryGroup.get_id().equals(finalQueryGroupId)); + + if (!exists) { + queryGroupId = QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(); + } + + if (task instanceof SearchShardTask) { + queryGroupsStateAccessor.getQueryGroupState(queryGroupId).shardCompletions.inc(); + } else { + queryGroupsStateAccessor.getQueryGroupState(queryGroupId).completions.inc(); } } } diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java index a1cb766579d43..97c48bd828978 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupTask.java @@ -17,7 +17,6 @@ import org.opensearch.tasks.CancellableTask; import java.util.Map; -import java.util.Optional; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -82,9 +81,11 @@ public final String getQueryGroupId() { * @param threadContext current threadContext */ public final void setQueryGroupId(final ThreadContext threadContext) { - this.queryGroupId = Optional.ofNullable(threadContext) - .map(threadContext1 -> threadContext1.getHeader(QUERY_GROUP_ID_HEADER)) - .orElse(DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); + if (threadContext != null && threadContext.getHeader(QUERY_GROUP_ID_HEADER) != null) { + this.queryGroupId = threadContext.getHeader(QUERY_GROUP_ID_HEADER); + } else { + this.queryGroupId = DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(); + } } public long getElapsedTime() { diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupsStateAccessor.java b/server/src/main/java/org/opensearch/wlm/QueryGroupsStateAccessor.java new file mode 100644 index 0000000000000..7f93e41f12092 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupsStateAccessor.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm; + +import org.opensearch.wlm.stats.QueryGroupState; + +import java.util.HashMap; +import java.util.Map; + +/** + * This class is used to decouple {@link QueryGroupService} and {@link org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService} to share the + * {@link QueryGroupState}s + */ +public class QueryGroupsStateAccessor { + // This map does not need to be concurrent since we will process the cluster state change serially and update + // this map with new additions and deletions of entries. QueryGroupState is thread safe + private final Map queryGroupStateMap; + + public QueryGroupsStateAccessor() { + this(new HashMap<>()); + } + + public QueryGroupsStateAccessor(Map queryGroupStateMap) { + this.queryGroupStateMap = queryGroupStateMap; + } + + /** + * returns the query groups state + */ + public Map getQueryGroupStateMap() { + return queryGroupStateMap; + } + + /** + * return QueryGroupState for the given queryGroupId + * @param queryGroupId + * @return QueryGroupState for the given queryGroupId, if id is invalid return default query group state + */ + public QueryGroupState getQueryGroupState(String queryGroupId) { + return queryGroupStateMap.getOrDefault(queryGroupId, queryGroupStateMap.get(QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get())); + } + + /** + * adds new QueryGroupState against given queryGroupId + * @param queryGroupId + */ + public void addNewQueryGroup(String queryGroupId) { + this.queryGroupStateMap.putIfAbsent(queryGroupId, new QueryGroupState()); + } + + /** + * removes QueryGroupState against given queryGroupId + * @param queryGroupId + */ + public void removeQueryGroup(String queryGroupId) { + this.queryGroupStateMap.remove(queryGroupId); + } +} diff --git a/server/src/main/java/org/opensearch/wlm/WlmMode.java b/server/src/main/java/org/opensearch/wlm/WlmMode.java new file mode 100644 index 0000000000000..40407525cc24d --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/WlmMode.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm; + +import org.opensearch.common.annotation.PublicApi; + +/** + * Enum to hold the values whether wlm is enabled or not + */ +@PublicApi(since = "2.18.0") +public enum WlmMode { + ENABLED("enabled"), + MONITOR_ONLY("monitor_only"), + DISABLED("disabled"); + + private final String name; + + WlmMode(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static WlmMode fromName(String name) { + for (WlmMode wlmMode : values()) { + if (wlmMode.getName().equals(name)) { + return wlmMode; + } + } + throw new IllegalArgumentException(name + " is an invalid WlmMode"); + } +} diff --git a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java index b3577c1b3219d..af25eedd7eed5 100644 --- a/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java +++ b/server/src/main/java/org/opensearch/wlm/WorkloadManagementSettings.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; /** * Main class to declare Workload Management related settings @@ -22,16 +23,66 @@ public class WorkloadManagementSettings { private static final Double DEFAULT_NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD = 0.9; private static final Double DEFAULT_NODE_LEVEL_CPU_REJECTION_THRESHOLD = 0.8; private static final Double DEFAULT_NODE_LEVEL_CPU_CANCELLATION_THRESHOLD = 0.9; + private static final Long DEFAULT_QUERYGROUP_SERVICE_RUN_INTERVAL_MILLIS = 1000L; public static final double NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD_MAX_VALUE = 0.95; public static final double NODE_LEVEL_MEMORY_REJECTION_THRESHOLD_MAX_VALUE = 0.9; public static final double NODE_LEVEL_CPU_CANCELLATION_THRESHOLD_MAX_VALUE = 0.95; public static final double NODE_LEVEL_CPU_REJECTION_THRESHOLD_MAX_VALUE = 0.9; + public static final String DEFAULT_WLM_MODE = "monitor_only"; private Double nodeLevelMemoryCancellationThreshold; private Double nodeLevelMemoryRejectionThreshold; private Double nodeLevelCpuCancellationThreshold; private Double nodeLevelCpuRejectionThreshold; + /** + * Setting name for QueryGroupService node duress streak + */ + public static final String QUERYGROUP_DURESS_STREAK_SETTING_NAME = "wlm.query_group.duress_streak"; + private int duressStreak; + public static final Setting QUERYGROUP_SERVICE_DURESS_STREAK_SETTING = Setting.intSetting( + QUERYGROUP_DURESS_STREAK_SETTING_NAME, + 3, + 3, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + /** + * Setting name for Query Group Service run interval + */ + public static final String QUERYGROUP_ENFORCEMENT_INTERVAL_SETTING_NAME = "wlm.query_group.enforcement_interval"; + + private TimeValue queryGroupServiceRunInterval; + /** + * Setting to control the run interval of Query Group Service + */ + public static final Setting QUERYGROUP_SERVICE_RUN_INTERVAL_SETTING = Setting.longSetting( + QUERYGROUP_ENFORCEMENT_INTERVAL_SETTING_NAME, + DEFAULT_QUERYGROUP_SERVICE_RUN_INTERVAL_MILLIS, + 1000, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + /** + * WLM mode setting name + */ + public static final String WLM_MODE_SETTING_NAME = "wlm.query_group.mode"; + + private volatile WlmMode wlmMode; + + /** + * WLM mode setting, which determines which mode WLM is operating in + */ + public static final Setting WLM_MODE_SETTING = new Setting( + WLM_MODE_SETTING_NAME, + DEFAULT_WLM_MODE, + WlmMode::fromName, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + /** * Setting name for node level memory based rejection threshold for QueryGroup service */ @@ -91,10 +142,13 @@ public class WorkloadManagementSettings { * @param clusterSettings - QueryGroup cluster settings */ public WorkloadManagementSettings(Settings settings, ClusterSettings clusterSettings) { + this.wlmMode = WLM_MODE_SETTING.get(settings); nodeLevelMemoryCancellationThreshold = NODE_LEVEL_MEMORY_CANCELLATION_THRESHOLD.get(settings); nodeLevelMemoryRejectionThreshold = NODE_LEVEL_MEMORY_REJECTION_THRESHOLD.get(settings); nodeLevelCpuCancellationThreshold = NODE_LEVEL_CPU_CANCELLATION_THRESHOLD.get(settings); nodeLevelCpuRejectionThreshold = NODE_LEVEL_CPU_REJECTION_THRESHOLD.get(settings); + this.queryGroupServiceRunInterval = TimeValue.timeValueMillis(QUERYGROUP_SERVICE_RUN_INTERVAL_SETTING.get(settings)); + duressStreak = QUERYGROUP_SERVICE_DURESS_STREAK_SETTING.get(settings); ensureRejectionThresholdIsLessThanCancellation( nodeLevelMemoryRejectionThreshold, @@ -113,6 +167,57 @@ public WorkloadManagementSettings(Settings settings, ClusterSettings clusterSett clusterSettings.addSettingsUpdateConsumer(NODE_LEVEL_MEMORY_REJECTION_THRESHOLD, this::setNodeLevelMemoryRejectionThreshold); clusterSettings.addSettingsUpdateConsumer(NODE_LEVEL_CPU_CANCELLATION_THRESHOLD, this::setNodeLevelCpuCancellationThreshold); clusterSettings.addSettingsUpdateConsumer(NODE_LEVEL_CPU_REJECTION_THRESHOLD, this::setNodeLevelCpuRejectionThreshold); + clusterSettings.addSettingsUpdateConsumer(WLM_MODE_SETTING, this::setWlmMode); + clusterSettings.addSettingsUpdateConsumer(QUERYGROUP_SERVICE_RUN_INTERVAL_SETTING, this::setQueryGroupServiceRunInterval); + clusterSettings.addSettingsUpdateConsumer(QUERYGROUP_SERVICE_DURESS_STREAK_SETTING, this::setDuressStreak); + } + + /** + * node duress streak getter + * @return current duressStreak value + */ + public int getDuressStreak() { + return duressStreak; + } + + /** + * node duress streak setter + * @param duressStreak new value + */ + private void setDuressStreak(int duressStreak) { + this.duressStreak = duressStreak; + } + + /** + * queryGroupServiceRunInterval setter + * @param newIntervalInMillis new value + */ + private void setQueryGroupServiceRunInterval(long newIntervalInMillis) { + this.queryGroupServiceRunInterval = TimeValue.timeValueMillis(newIntervalInMillis); + } + + /** + * queryGroupServiceRunInterval getter + * @return current queryGroupServiceRunInterval value + */ + public TimeValue getQueryGroupServiceRunInterval() { + return this.queryGroupServiceRunInterval; + } + + /** + * WlmMode setter + * @param mode new mode value + */ + private void setWlmMode(final WlmMode mode) { + this.wlmMode = mode; + } + + /** + * WlmMode getter + * @return the current wlmMode + */ + public WlmMode getWlmMode() { + return this.wlmMode; } /** diff --git a/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java index a2c97c8d8635b..e82a19c5f7af2 100644 --- a/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java +++ b/server/src/main/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationService.java @@ -8,20 +8,26 @@ package org.opensearch.wlm.cancellation; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.cluster.metadata.QueryGroup; -import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.TaskCancellation; import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.QueryGroupsStateAccessor; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WlmMode; import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.stats.QueryGroupState; import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -47,46 +53,78 @@ */ public class QueryGroupTaskCancellationService { public static final double MIN_VALUE = 1e-9; + private static final Logger log = LogManager.getLogger(QueryGroupTaskCancellationService.class); private final WorkloadManagementSettings workloadManagementSettings; private final TaskSelectionStrategy taskSelectionStrategy; private final QueryGroupResourceUsageTrackerService resourceUsageTrackerService; // a map of QueryGroupId to its corresponding QueryGroupLevelResourceUsageView object Map queryGroupLevelResourceUsageViews; - private final Collection activeQueryGroups; - private final Collection deletedQueryGroups; + private final QueryGroupsStateAccessor queryGroupStateAccessor; public QueryGroupTaskCancellationService( WorkloadManagementSettings workloadManagementSettings, TaskSelectionStrategy taskSelectionStrategy, QueryGroupResourceUsageTrackerService resourceUsageTrackerService, - Collection activeQueryGroups, - Collection deletedQueryGroups + QueryGroupsStateAccessor queryGroupStateAccessor ) { this.workloadManagementSettings = workloadManagementSettings; this.taskSelectionStrategy = taskSelectionStrategy; this.resourceUsageTrackerService = resourceUsageTrackerService; - this.activeQueryGroups = activeQueryGroups; - this.deletedQueryGroups = deletedQueryGroups; + this.queryGroupStateAccessor = queryGroupStateAccessor; } /** * Cancel tasks based on the implemented strategy. */ - public final void cancelTasks(BooleanSupplier isNodeInDuress) { + public void cancelTasks( + BooleanSupplier isNodeInDuress, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { queryGroupLevelResourceUsageViews = resourceUsageTrackerService.constructQueryGroupLevelUsageViews(); // cancel tasks from QueryGroups that are in Enforced mode that are breaching their resource limits - cancelTasks(ResiliencyMode.ENFORCED); + cancelTasks(ResiliencyMode.ENFORCED, activeQueryGroups); // if the node is in duress, cancel tasks accordingly. - handleNodeDuress(isNodeInDuress); + handleNodeDuress(isNodeInDuress, activeQueryGroups, deletedQueryGroups); + + updateResourceUsageInQueryGroupState(activeQueryGroups); + } + + private void updateResourceUsageInQueryGroupState(Collection activeQueryGroups) { + Set isSearchWorkloadRunning = new HashSet<>(); + for (Map.Entry queryGroupLevelResourceUsageViewEntry : queryGroupLevelResourceUsageViews + .entrySet()) { + isSearchWorkloadRunning.add(queryGroupLevelResourceUsageViewEntry.getKey()); + QueryGroupState queryGroupState = getQueryGroupState(queryGroupLevelResourceUsageViewEntry.getKey()); + TRACKED_RESOURCES.forEach(resourceType -> { + final double currentUsage = queryGroupLevelResourceUsageViewEntry.getValue().getResourceUsageData().get(resourceType); + queryGroupState.getResourceState().get(resourceType).setLastRecordedUsage(currentUsage); + }); + } + + activeQueryGroups.forEach(queryGroup -> { + if (!isSearchWorkloadRunning.contains(queryGroup.get_id())) { + TRACKED_RESOURCES.forEach( + resourceType -> getQueryGroupState(queryGroup.get_id()).getResourceState().get(resourceType).setLastRecordedUsage(0.0) + ); + } + }); } - private void handleNodeDuress(BooleanSupplier isNodeInDuress) { + private void handleNodeDuress( + BooleanSupplier isNodeInDuress, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { if (!isNodeInDuress.getAsBoolean()) { return; } // List of tasks to be executed in order if the node is in duress - List> duressActions = List.of(v -> cancelTasksFromDeletedQueryGroups(), v -> cancelTasks(ResiliencyMode.SOFT)); + List> duressActions = List.of( + v -> cancelTasksFromDeletedQueryGroups(deletedQueryGroups), + v -> cancelTasks(ResiliencyMode.SOFT, activeQueryGroups) + ); for (Consumer duressAction : duressActions) { if (!isNodeInDuress.getAsBoolean()) { @@ -96,8 +134,8 @@ private void handleNodeDuress(BooleanSupplier isNodeInDuress) { } } - private void cancelTasksFromDeletedQueryGroups() { - cancelTasks(getAllCancellableTasks(this.deletedQueryGroups)); + private void cancelTasksFromDeletedQueryGroups(Collection deletedQueryGroups) { + cancelTasks(getAllCancellableTasks(deletedQueryGroups)); } /** @@ -105,9 +143,9 @@ private void cancelTasksFromDeletedQueryGroups() { * * @return List of tasks that can be cancelled */ - List getAllCancellableTasks(ResiliencyMode resiliencyMode) { + List getAllCancellableTasks(ResiliencyMode resiliencyMode, Collection queryGroups) { return getAllCancellableTasks( - activeQueryGroups.stream().filter(queryGroup -> queryGroup.getResiliencyMode() == resiliencyMode).collect(Collectors.toList()) + queryGroups.stream().filter(queryGroup -> queryGroup.getResiliencyMode() == resiliencyMode).collect(Collectors.toList()) ); } @@ -118,6 +156,7 @@ List getAllCancellableTasks(ResiliencyMode resiliencyMode) { */ List getAllCancellableTasks(Collection queryGroups) { List taskCancellations = new ArrayList<>(); + final List onCancelCallbacks = new ArrayList<>(); for (QueryGroup queryGroup : queryGroups) { final List reasons = new ArrayList<>(); List selectedTasks = new ArrayList<>(); @@ -127,8 +166,7 @@ List getAllCancellableTasks(Collection queryGroups .calculateResourceUsage(selectedTasks); if (excessUsage > MIN_VALUE) { reasons.add(new TaskCancellation.Reason(generateReasonString(queryGroup, resourceType), 1)); - // TODO: We will need to add the cancellation callback for these resources for the queryGroup to reflect stats - + onCancelCallbacks.add(this.getResourceTypeOnCancelCallback(queryGroup.get_id(), resourceType)); // Only add tasks not already added to avoid double cancellations selectedTasks.addAll( taskSelectionStrategy.selectTasksForCancellation(getTasksFor(queryGroup), excessUsage, resourceType) @@ -140,8 +178,9 @@ List getAllCancellableTasks(Collection queryGroups } if (!reasons.isEmpty()) { + onCancelCallbacks.add(getQueryGroupState(queryGroup.get_id()).totalCancellations::inc); taskCancellations.addAll( - selectedTasks.stream().map(task -> createTaskCancellation(task, reasons)).collect(Collectors.toList()) + selectedTasks.stream().map(task -> new TaskCancellation(task, reasons, onCancelCallbacks)).collect(Collectors.toList()) ); } } @@ -164,16 +203,27 @@ private List getTasksFor(QueryGroup queryGroup) { return queryGroupLevelResourceUsageViews.get(queryGroup.get_id()).getActiveTasks(); } - private void cancelTasks(ResiliencyMode resiliencyMode) { - cancelTasks(getAllCancellableTasks(resiliencyMode)); + private void cancelTasks(ResiliencyMode resiliencyMode, Collection queryGroups) { + cancelTasks(getAllCancellableTasks(resiliencyMode, queryGroups)); } private void cancelTasks(List cancellableTasks) { - cancellableTasks.forEach(TaskCancellation::cancel); - } - private TaskCancellation createTaskCancellation(CancellableTask task, List reasons) { - return new TaskCancellation(task, reasons, List.of(this::callbackOnCancel)); + Consumer cancellationLoggingConsumer = (taskCancellation -> { + log.warn( + "Task {} is eligible for cancellation for reason {}", + taskCancellation.getTask().getId(), + taskCancellation.getReasonString() + ); + }); + Consumer cancellationConsumer = cancellationLoggingConsumer; + if (workloadManagementSettings.getWlmMode() == WlmMode.ENABLED) { + cancellationConsumer = (taskCancellation -> { + cancellationLoggingConsumer.accept(taskCancellation); + taskCancellation.cancel(); + }); + } + cancellableTasks.forEach(cancellationConsumer); } private double getExcessUsage(QueryGroup queryGroup, ResourceType resourceType) { @@ -199,7 +249,26 @@ private double getNormalisedThreshold(QueryGroup queryGroup, ResourceType resour return queryGroup.getResourceLimits().get(resourceType) * nodeLevelCancellationThreshold; } - private void callbackOnCancel() { - // TODO Implement callback logic here mostly used for Stats + private Runnable getResourceTypeOnCancelCallback(String queryGroupId, ResourceType resourceType) { + QueryGroupState queryGroupState = getQueryGroupState(queryGroupId); + return queryGroupState.getResourceState().get(resourceType).cancellations::inc; + } + + private QueryGroupState getQueryGroupState(String queryGroupId) { + assert queryGroupId != null : "queryGroupId should never be null at this point."; + + return queryGroupStateAccessor.getQueryGroupState(queryGroupId); + } + + /** + * Removes the queryGroups from deleted list if it doesn't have any tasks running + */ + public void pruneDeletedQueryGroups(Collection deletedQueryGroups) { + List currentDeletedQueryGroups = new ArrayList<>(deletedQueryGroups); + for (QueryGroup queryGroup : currentDeletedQueryGroups) { + if (queryGroupLevelResourceUsageViews.get(queryGroup.get_id()).getActiveTasks().isEmpty()) { + deletedQueryGroups.remove(queryGroup); + } + } } } diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java index 376d34dd7c8ca..cbc7046a79464 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -19,14 +19,19 @@ */ public class QueryGroupState { /** - * completions at the query group level, this is a cumulative counter since the Opensearch start time + * co-ordinator level completions at the query group level, this is a cumulative counter since the Opensearch start time */ - final CounterMetric completions = new CounterMetric(); + public final CounterMetric completions = new CounterMetric(); + + /** + * shard level completions at the query group level, this is a cumulative counter since the Opensearch start time + */ + public final CounterMetric shardCompletions = new CounterMetric(); /** * rejections at the query group level, this is a cumulative counter since the OpenSearch start time */ - final CounterMetric totalRejections = new CounterMetric(); + public final CounterMetric totalRejections = new CounterMetric(); /** * this will track the cumulative failures in a query group @@ -36,7 +41,7 @@ public class QueryGroupState { /** * This will track total number of cancellations in the query group due to all resource type breaches */ - final CounterMetric totalCancellations = new CounterMetric(); + public final CounterMetric totalCancellations = new CounterMetric(); /** * This is used to store the resource type state both for CPU and MEMORY @@ -54,12 +59,20 @@ public QueryGroupState() { /** * - * @return completions in the query group + * @return co-ordinator completions in the query group */ public long getCompletions() { return completions.count(); } + /** + * + * @return shard completions in the query group + */ + public long getShardCompletions() { + return shardCompletions.count(); + } + /** * * @return rejections in the query group @@ -92,9 +105,9 @@ public Map getResourceState() { * This class holds the resource level stats for the query group */ public static class ResourceTypeState { - final ResourceType resourceType; - final CounterMetric cancellations = new CounterMetric(); - final CounterMetric rejections = new CounterMetric(); + public final ResourceType resourceType; + public final CounterMetric cancellations = new CounterMetric(); + public final CounterMetric rejections = new CounterMetric(); private double lastRecordedUsage = 0; public ResourceTypeState(ResourceType resourceType) { diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 2b389c2167778..9d74201de252b 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -91,7 +91,9 @@ public static class QueryGroupStatsHolder implements ToXContentObject, Writeable public static final String REJECTIONS = "rejections"; public static final String TOTAL_CANCELLATIONS = "total_cancellations"; public static final String FAILURES = "failures"; + public static final String SHARD_COMPLETIONS = "shard_completions"; private long completions; + private long shardCompletions; private long rejections; private long failures; private long totalCancellations; @@ -105,11 +107,13 @@ public QueryGroupStatsHolder( long rejections, long failures, long totalCancellations, + long shardCompletions, Map resourceStats ) { this.completions = completions; this.rejections = rejections; this.failures = failures; + this.shardCompletions = shardCompletions; this.totalCancellations = totalCancellations; this.resourceStats = resourceStats; } @@ -119,6 +123,7 @@ public QueryGroupStatsHolder(StreamInput in) throws IOException { this.rejections = in.readVLong(); this.failures = in.readVLong(); this.totalCancellations = in.readVLong(); + this.shardCompletions = in.readVLong(); this.resourceStats = in.readMap((i) -> ResourceType.fromName(i.readString()), ResourceStats::new); } @@ -140,6 +145,7 @@ public static QueryGroupStatsHolder from(QueryGroupState queryGroupState) { statsHolder.rejections = queryGroupState.getTotalRejections(); statsHolder.failures = queryGroupState.getFailures(); statsHolder.totalCancellations = queryGroupState.getTotalCancellations(); + statsHolder.shardCompletions = queryGroupState.getShardCompletions(); statsHolder.resourceStats = resourceStatsMap; return statsHolder; } @@ -155,6 +161,7 @@ public static void writeTo(StreamOutput out, QueryGroupStatsHolder statsHolder) out.writeVLong(statsHolder.rejections); out.writeVLong(statsHolder.failures); out.writeVLong(statsHolder.totalCancellations); + out.writeVLong(statsHolder.shardCompletions); out.writeMap(statsHolder.resourceStats, (o, val) -> o.writeString(val.getName()), ResourceStats::writeTo); } @@ -166,6 +173,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(COMPLETIONS, completions); + builder.field(SHARD_COMPLETIONS, shardCompletions); builder.field(REJECTIONS, rejections); builder.field(FAILURES, failures); builder.field(TOTAL_CANCELLATIONS, totalCancellations); @@ -187,6 +195,7 @@ public boolean equals(Object o) { QueryGroupStatsHolder that = (QueryGroupStatsHolder) o; return completions == that.completions && rejections == that.rejections + && shardCompletions == that.shardCompletions && Objects.equals(resourceStats, that.resourceStats) && failures == that.failures && totalCancellations == that.totalCancellations; @@ -194,7 +203,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(completions, rejections, totalCancellations, failures, resourceStats); + return Objects.hash(completions, shardCompletions, rejections, totalCancellations, failures, resourceStats); } } diff --git a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java index b23d9ff342139..19f7bf48d8421 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java @@ -47,6 +47,8 @@ public Map constructQueryGroupLevelUsa // Iterate over each QueryGroup entry for (Map.Entry> queryGroupEntry : tasksByQueryGroup.entrySet()) { + // refresh the resource stats + taskResourceTrackingService.refreshResourceStats(queryGroupEntry.getValue().toArray(new QueryGroupTask[0])); // Compute the QueryGroup resource usage final Map queryGroupUsage = new EnumMap<>(ResourceType.class); for (ResourceType resourceType : TRACKED_RESOURCES) { diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index a444eb42eac2e..8c4685bee6a0b 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -39,6 +39,8 @@ import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.QueryGroupService; +import org.opensearch.wlm.QueryGroupTask; import org.opensearch.wlm.ResourceType; import org.junit.After; import org.junit.Before; @@ -75,10 +77,12 @@ public class SearchBackpressureServiceTests extends OpenSearchTestCase { MockTransportService transportService; TaskManager taskManager; ThreadPool threadPool; + QueryGroupService queryGroupService; @Before public void setup() { threadPool = new TestThreadPool(getClass().getName()); + queryGroupService = mock(QueryGroupService.class); transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, NoopTracer.INSTANCE); transportService.start(); transportService.acceptIncomingRequests(); @@ -120,9 +124,12 @@ public void testIsNodeInDuress() { new NodeDuressTrackers(duressTrackers), new TaskResourceUsageTrackers(), new TaskResourceUsageTrackers(), - taskManager + taskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + // Node not in duress. cpuUsage.set(0.0); heapUsage.set(0.0); @@ -163,9 +170,12 @@ public void testTrackerStateUpdateOnSearchTaskCompletion() { new NodeDuressTrackers(new EnumMap<>(ResourceType.class)), taskResourceUsageTrackers, new TaskResourceUsageTrackers(), - taskManager + taskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + for (int i = 0; i < 100; i++) { // service.onTaskCompleted(new SearchTask(1, "test", "test", () -> "Test", TaskId.EMPTY_TASK_ID, new HashMap<>())); service.onTaskCompleted(createMockTaskWithResourceStats(SearchTask.class, 100, 200, i)); @@ -194,9 +204,12 @@ public void testTrackerStateUpdateOnSearchShardTaskCompletion() { new NodeDuressTrackers(new EnumMap<>(ResourceType.class)), new TaskResourceUsageTrackers(), taskResourceUsageTrackers, - taskManager + taskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + // Record task completions to update the tracker state. Tasks other than SearchTask & SearchShardTask are ignored. service.onTaskCompleted(createMockTaskWithResourceStats(CancellableTask.class, 100, 200, 101)); for (int i = 0; i < 100; i++) { @@ -246,9 +259,12 @@ public void testSearchTaskInFlightCancellation() { new NodeDuressTrackers(duressTrackers), taskResourceUsageTrackers, new TaskResourceUsageTrackers(), - mockTaskManager + mockTaskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + // Run two iterations so that node is marked 'in duress' from the third iteration onwards. service.doRun(); service.doRun(); @@ -261,7 +277,7 @@ public void testSearchTaskInFlightCancellation() { when(settings.getSearchTaskSettings()).thenReturn(searchTaskSettings); // Create a mix of low and high resource usage SearchTasks (50 low + 25 high resource usage tasks). - Map activeSearchTasks = new HashMap<>(); + Map activeSearchTasks = new HashMap<>(); for (long i = 0; i < 75; i++) { if (i % 3 == 0) { activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 500, taskHeapUsageBytes, i)); @@ -269,6 +285,7 @@ public void testSearchTaskInFlightCancellation() { activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, taskHeapUsageBytes, i)); } } + activeSearchTasks.values().forEach(task -> task.setQueryGroupId(threadPool.getThreadContext())); doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks(); // There are 25 SearchTasks eligible for cancellation but only 5 will be cancelled (burst limit). @@ -338,9 +355,12 @@ public void testSearchShardTaskInFlightCancellation() { nodeDuressTrackers, new TaskResourceUsageTrackers(), taskResourceUsageTrackers, - mockTaskManager + mockTaskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + // Run two iterations so that node is marked 'in duress' from the third iteration onwards. service.doRun(); service.doRun(); @@ -353,7 +373,7 @@ public void testSearchShardTaskInFlightCancellation() { when(settings.getSearchShardTaskSettings()).thenReturn(searchShardTaskSettings); // Create a mix of low and high resource usage tasks (60 low + 15 high resource usage tasks). - Map activeSearchShardTasks = new HashMap<>(); + Map activeSearchShardTasks = new HashMap<>(); for (long i = 0; i < 75; i++) { if (i % 5 == 0) { activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 500, taskHeapUsageBytes, i)); @@ -361,6 +381,7 @@ public void testSearchShardTaskInFlightCancellation() { activeSearchShardTasks.put(i, createMockTaskWithResourceStats(SearchShardTask.class, 100, taskHeapUsageBytes, i)); } } + activeSearchShardTasks.values().forEach(task -> task.setQueryGroupId(threadPool.getThreadContext())); doReturn(activeSearchShardTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks(); // There are 15 SearchShardTasks eligible for cancellation but only 10 will be cancelled (burst limit). @@ -439,9 +460,12 @@ public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() { nodeDuressTrackers, taskResourceUsageTrackers, new TaskResourceUsageTrackers(), - mockTaskManager + mockTaskManager, + queryGroupService ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); + service.doRun(); service.doRun(); @@ -451,7 +475,7 @@ public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() { when(settings.getSearchTaskSettings()).thenReturn(searchTaskSettings); // Create a mix of low and high resource usage tasks (60 low + 15 high resource usage tasks). - Map activeSearchTasks = new HashMap<>(); + Map activeSearchTasks = new HashMap<>(); for (long i = 0; i < 75; i++) { if (i % 5 == 0) { activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 500, 800, i)); @@ -459,6 +483,7 @@ public void testNonCancellationOfHeapBasedTasksWhenHeapNotInDuress() { activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, 800, i)); } } + activeSearchTasks.values().forEach(task -> task.setQueryGroupId(threadPool.getThreadContext())); doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks(); // this will trigger cancellation but these cancellation should only be cpu based @@ -534,10 +559,12 @@ public void testNonCancellationWhenSearchTrafficIsNotQualifyingForCancellation() nodeDuressTrackers, taskResourceUsageTrackers, new TaskResourceUsageTrackers(), - mockTaskManager + mockTaskManager, + queryGroupService ) ); + when(queryGroupService.shouldSBPHandle(any())).thenReturn(true); when(service.isHeapUsageDominatedBySearch(anyList(), anyDouble())).thenReturn(false); service.doRun(); @@ -549,15 +576,16 @@ public void testNonCancellationWhenSearchTrafficIsNotQualifyingForCancellation() when(settings.getSearchTaskSettings()).thenReturn(searchTaskSettings); // Create a mix of low and high resource usage tasks (60 low + 15 high resource usage tasks). - Map activeSearchTasks = new HashMap<>(); + Map activeSearchTasks = new HashMap<>(); for (long i = 0; i < 75; i++) { - Class taskType = randomBoolean() ? SearchTask.class : SearchShardTask.class; + Class taskType = randomBoolean() ? SearchTask.class : SearchShardTask.class; if (i % 5 == 0) { activeSearchTasks.put(i, createMockTaskWithResourceStats(taskType, 500, 800, i)); } else { activeSearchTasks.put(i, createMockTaskWithResourceStats(taskType, 100, 800, i)); } } + activeSearchTasks.values().forEach(task -> task.setQueryGroupId(threadPool.getThreadContext())); doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks(); // this will trigger cancellation but the cancellation should not happen as the node is not is duress because of search traffic diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java new file mode 100644 index 0000000000000..c5cf0dac4f807 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java @@ -0,0 +1,489 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm; + +import org.opensearch.action.search.SearchTask; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.QueryGroup; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.search.backpressure.trackers.NodeDuressTrackers; +import org.opensearch.tasks.Task; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.Scheduler; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService; +import org.opensearch.wlm.cancellation.TaskSelectionStrategy; +import org.opensearch.wlm.stats.QueryGroupState; +import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.BooleanSupplier; + +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import static org.opensearch.wlm.tracker.ResourceUsageCalculatorTests.createMockTaskWithResourceStats; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class QueryGroupServiceTests extends OpenSearchTestCase { + private QueryGroupService queryGroupService; + private QueryGroupTaskCancellationService mockCancellationService; + private ClusterService mockClusterService; + private ThreadPool mockThreadPool; + private WorkloadManagementSettings mockWorkloadManagementSettings; + private Scheduler.Cancellable mockScheduledFuture; + private Map mockQueryGroupStateMap; + NodeDuressTrackers mockNodeDuressTrackers; + QueryGroupsStateAccessor mockQueryGroupsStateAccessor; + + public void setUp() throws Exception { + super.setUp(); + mockClusterService = Mockito.mock(ClusterService.class); + mockThreadPool = Mockito.mock(ThreadPool.class); + mockScheduledFuture = Mockito.mock(Scheduler.Cancellable.class); + mockWorkloadManagementSettings = Mockito.mock(WorkloadManagementSettings.class); + mockQueryGroupStateMap = new HashMap<>(); + mockNodeDuressTrackers = Mockito.mock(NodeDuressTrackers.class); + mockCancellationService = Mockito.mock(TestQueryGroupCancellationService.class); + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + new HashSet<>(), + new HashSet<>() + ); + } + + public void tearDown() throws Exception { + super.tearDown(); + mockThreadPool.shutdown(); + } + + public void testClusterChanged() { + ClusterChangedEvent mockClusterChangedEvent = Mockito.mock(ClusterChangedEvent.class); + ClusterState mockPreviousClusterState = Mockito.mock(ClusterState.class); + ClusterState mockClusterState = Mockito.mock(ClusterState.class); + Metadata mockPreviousMetadata = Mockito.mock(Metadata.class); + Metadata mockMetadata = Mockito.mock(Metadata.class); + QueryGroup addedQueryGroup = new QueryGroup( + "addedQueryGroup", + "4242", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.MEMORY, 0.5)), + 1L + ); + QueryGroup deletedQueryGroup = new QueryGroup( + "deletedQueryGroup", + "4241", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.MEMORY, 0.5)), + 1L + ); + Map previousQueryGroups = new HashMap<>(); + previousQueryGroups.put("4242", addedQueryGroup); + Map currentQueryGroups = new HashMap<>(); + currentQueryGroups.put("4241", deletedQueryGroup); + + when(mockClusterChangedEvent.previousState()).thenReturn(mockPreviousClusterState); + when(mockClusterChangedEvent.state()).thenReturn(mockClusterState); + when(mockPreviousClusterState.metadata()).thenReturn(mockPreviousMetadata); + when(mockClusterState.metadata()).thenReturn(mockMetadata); + when(mockPreviousMetadata.queryGroups()).thenReturn(previousQueryGroups); + when(mockMetadata.queryGroups()).thenReturn(currentQueryGroups); + queryGroupService.clusterChanged(mockClusterChangedEvent); + + Set currentQueryGroupsExpected = Set.of(currentQueryGroups.get("4241")); + Set previousQueryGroupsExpected = Set.of(previousQueryGroups.get("4242")); + + assertEquals(currentQueryGroupsExpected, queryGroupService.getActiveQueryGroups()); + assertEquals(previousQueryGroupsExpected, queryGroupService.getDeletedQueryGroups()); + } + + public void testDoStart_SchedulesTask() { + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + when(mockWorkloadManagementSettings.getQueryGroupServiceRunInterval()).thenReturn(TimeValue.timeValueSeconds(1)); + queryGroupService.doStart(); + Mockito.verify(mockThreadPool).scheduleWithFixedDelay(any(Runnable.class), any(TimeValue.class), eq(ThreadPool.Names.GENERIC)); + } + + public void testDoStop_CancelsScheduledTask() { + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + when(mockThreadPool.scheduleWithFixedDelay(any(), any(), any())).thenReturn(mockScheduledFuture); + queryGroupService.doStart(); + queryGroupService.doStop(); + Mockito.verify(mockScheduledFuture).cancel(); + } + + public void testDoRun_WhenModeEnabled() { + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + when(mockNodeDuressTrackers.isNodeInDuress()).thenReturn(true); + // Call the method + queryGroupService.doRun(); + + // Verify that refreshQueryGroups was called + + // Verify that cancelTasks was called with a BooleanSupplier + ArgumentCaptor booleanSupplierCaptor = ArgumentCaptor.forClass(BooleanSupplier.class); + Mockito.verify(mockCancellationService).cancelTasks(booleanSupplierCaptor.capture(), any(), any()); + + // Assert the behavior of the BooleanSupplier + BooleanSupplier capturedSupplier = booleanSupplierCaptor.getValue(); + assertTrue(capturedSupplier.getAsBoolean()); + + } + + public void testDoRun_WhenModeDisabled() { + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.DISABLED); + when(mockNodeDuressTrackers.isNodeInDuress()).thenReturn(false); + queryGroupService.doRun(); + // Verify that refreshQueryGroups was called + + Mockito.verify(mockCancellationService, never()).cancelTasks(any(), any(), any()); + + } + + public void testRejectIfNeeded_whenQueryGroupIdIsNullOrDefaultOne() { + QueryGroup testQueryGroup = new QueryGroup( + "testQueryGroup", + "queryGroupId1", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.10)), + 1L + ); + Set activeQueryGroups = new HashSet<>() { + { + add(testQueryGroup); + } + }; + mockQueryGroupStateMap = new HashMap<>(); + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + mockQueryGroupStateMap.put("queryGroupId1", new QueryGroupState()); + + Map spyMap = spy(mockQueryGroupStateMap); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + queryGroupService.rejectIfNeeded(null); + + verify(spyMap, never()).get(any()); + + queryGroupService.rejectIfNeeded(QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); + verify(spyMap, never()).get(any()); + } + + public void testRejectIfNeeded_whenQueryGroupIsSoftMode() { + QueryGroup testQueryGroup = new QueryGroup( + "testQueryGroup", + "queryGroupId1", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.SOFT, Map.of(ResourceType.CPU, 0.10)), + 1L + ); + Set activeQueryGroups = new HashSet<>() { + { + add(testQueryGroup); + } + }; + mockQueryGroupStateMap = new HashMap<>(); + QueryGroupState spyState = spy(new QueryGroupState()); + mockQueryGroupStateMap.put("queryGroupId1", spyState); + + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + + Map spyMap = spy(mockQueryGroupStateMap); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + queryGroupService.rejectIfNeeded("queryGroupId1"); + + verify(spyState, never()).getResourceState(); + } + + public void testRejectIfNeeded_whenQueryGroupIsEnforcedMode_andNotBreaching() { + QueryGroup testQueryGroup = new QueryGroup( + "testQueryGroup", + "queryGroupId1", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.10)), + 1L + ); + QueryGroup spuQueryGroup = spy(testQueryGroup); + Set activeQueryGroups = new HashSet<>() { + { + add(spuQueryGroup); + } + }; + mockQueryGroupStateMap = new HashMap<>(); + QueryGroupState queryGroupState = new QueryGroupState(); + queryGroupState.getResourceState().get(ResourceType.CPU).setLastRecordedUsage(0.05); + + mockQueryGroupStateMap.put("queryGroupId1", queryGroupState); + + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + when(mockWorkloadManagementSettings.getNodeLevelCpuRejectionThreshold()).thenReturn(0.8); + queryGroupService.rejectIfNeeded("queryGroupId1"); + + // verify the check to compare the current usage and limit + // this should happen 3 times => 2 to check whether the resource limit has the TRACKED resource type and 1 to get the value + verify(spuQueryGroup, times(3)).getResourceLimits(); + assertEquals(0, queryGroupState.getResourceState().get(ResourceType.CPU).rejections.count()); + assertEquals(0, queryGroupState.totalRejections.count()); + } + + public void testRejectIfNeeded_whenQueryGroupIsEnforcedMode_andBreaching() { + QueryGroup testQueryGroup = new QueryGroup( + "testQueryGroup", + "queryGroupId1", + new MutableQueryGroupFragment( + MutableQueryGroupFragment.ResiliencyMode.ENFORCED, + Map.of(ResourceType.CPU, 0.10, ResourceType.MEMORY, 0.10) + ), + 1L + ); + QueryGroup spuQueryGroup = spy(testQueryGroup); + Set activeQueryGroups = new HashSet<>() { + { + add(spuQueryGroup); + } + }; + mockQueryGroupStateMap = new HashMap<>(); + QueryGroupState queryGroupState = new QueryGroupState(); + queryGroupState.getResourceState().get(ResourceType.CPU).setLastRecordedUsage(0.18); + queryGroupState.getResourceState().get(ResourceType.MEMORY).setLastRecordedUsage(0.18); + QueryGroupState spyState = spy(queryGroupState); + + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + + mockQueryGroupStateMap.put("queryGroupId1", spyState); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + assertThrows(OpenSearchRejectedExecutionException.class, () -> queryGroupService.rejectIfNeeded("queryGroupId1")); + + // verify the check to compare the current usage and limit + // this should happen 3 times => 1 to check whether the resource limit has the TRACKED resource type and 1 to get the value + // because it will break out of the loop since the limits are breached + verify(spuQueryGroup, times(2)).getResourceLimits(); + assertEquals( + 1, + queryGroupState.getResourceState().get(ResourceType.CPU).rejections.count() + queryGroupState.getResourceState() + .get(ResourceType.MEMORY).rejections.count() + ); + assertEquals(1, queryGroupState.totalRejections.count()); + } + + public void testRejectIfNeeded_whenFeatureIsNotEnabled() { + QueryGroup testQueryGroup = new QueryGroup( + "testQueryGroup", + "queryGroupId1", + new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.10)), + 1L + ); + Set activeQueryGroups = new HashSet<>() { + { + add(testQueryGroup); + } + }; + mockQueryGroupStateMap = new HashMap<>(); + mockQueryGroupStateMap.put("queryGroupId1", new QueryGroupState()); + + Map spyMap = spy(mockQueryGroupStateMap); + + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.DISABLED); + + queryGroupService.rejectIfNeeded(testQueryGroup.get_id()); + verify(spyMap, never()).get(any()); + } + + public void testOnTaskCompleted() { + Task task = createMockTaskWithResourceStats(SearchTask.class, 100, 200, 0, 12); + mockThreadPool = new TestThreadPool("queryGroupServiceTests"); + mockThreadPool.getThreadContext().putHeader(QueryGroupTask.QUERY_GROUP_ID_HEADER, "testId"); + QueryGroupState queryGroupState = new QueryGroupState(); + mockQueryGroupStateMap.put("testId", queryGroupState); + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + new HashSet<>() { + { + add( + new QueryGroup( + "testQueryGroup", + "testId", + new MutableQueryGroupFragment( + MutableQueryGroupFragment.ResiliencyMode.ENFORCED, + Map.of(ResourceType.CPU, 0.10, ResourceType.MEMORY, 0.10) + ), + 1L + ) + ); + } + }, + new HashSet<>() + ); + + ((QueryGroupTask) task).setQueryGroupId(mockThreadPool.getThreadContext()); + queryGroupService.onTaskCompleted(task); + + assertEquals(1, queryGroupState.completions.count()); + + // test non QueryGroupTask + task = new Task(1, "simple", "test", "mock task", null, null); + queryGroupService.onTaskCompleted(task); + + // It should still be 1 + assertEquals(1, queryGroupState.completions.count()); + + mockThreadPool.shutdown(); + } + + public void testShouldSBPHandle() { + QueryGroupTask task = createMockTaskWithResourceStats(SearchTask.class, 100, 200, 0, 12); + QueryGroupState queryGroupState = new QueryGroupState(); + Set activeQueryGroups = new HashSet<>(); + mockQueryGroupStateMap.put("testId", queryGroupState); + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + Collections.emptySet() + ); + + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + + // Default queryGroupId + mockThreadPool = new TestThreadPool("queryGroupServiceTests"); + mockThreadPool.getThreadContext() + .putHeader(QueryGroupTask.QUERY_GROUP_ID_HEADER, QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get()); + task.setQueryGroupId(mockThreadPool.getThreadContext()); + assertTrue(queryGroupService.shouldSBPHandle(task)); + + mockThreadPool.shutdownNow(); + + // invalid queryGroup task + mockThreadPool = new TestThreadPool("queryGroupServiceTests"); + mockThreadPool.getThreadContext().putHeader(QueryGroupTask.QUERY_GROUP_ID_HEADER, "testId"); + task.setQueryGroupId(mockThreadPool.getThreadContext()); + assertTrue(queryGroupService.shouldSBPHandle(task)); + + // Valid query group task but wlm not enabled + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.DISABLED); + activeQueryGroups.add( + new QueryGroup( + "testQueryGroup", + "testId", + new MutableQueryGroupFragment( + MutableQueryGroupFragment.ResiliencyMode.ENFORCED, + Map.of(ResourceType.CPU, 0.10, ResourceType.MEMORY, 0.10) + ), + 1L + ) + ); + assertTrue(queryGroupService.shouldSBPHandle(task)); + + } + + // This is needed to test the behavior of QueryGroupService#doRun method + static class TestQueryGroupCancellationService extends QueryGroupTaskCancellationService { + public TestQueryGroupCancellationService( + WorkloadManagementSettings workloadManagementSettings, + TaskSelectionStrategy taskSelectionStrategy, + QueryGroupResourceUsageTrackerService resourceUsageTrackerService, + QueryGroupsStateAccessor queryGroupsStateAccessor, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { + super(workloadManagementSettings, taskSelectionStrategy, resourceUsageTrackerService, queryGroupsStateAccessor); + } + + @Override + public void cancelTasks( + BooleanSupplier isNodeInDuress, + Collection activeQueryGroups, + Collection deletedQueryGroups + ) { + + } + } +} diff --git a/server/src/test/java/org/opensearch/wlm/WorkloadManagementTransportInterceptorTests.java b/server/src/test/java/org/opensearch/wlm/WorkloadManagementTransportInterceptorTests.java index 4668b845150a9..d4cd7b79455a3 100644 --- a/server/src/test/java/org/opensearch/wlm/WorkloadManagementTransportInterceptorTests.java +++ b/server/src/test/java/org/opensearch/wlm/WorkloadManagementTransportInterceptorTests.java @@ -8,24 +8,56 @@ package org.opensearch.wlm; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportRequestHandler; import org.opensearch.wlm.WorkloadManagementTransportInterceptor.RequestHandler; +import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService; + +import java.util.Collections; import static org.opensearch.threadpool.ThreadPool.Names.SAME; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class WorkloadManagementTransportInterceptorTests extends OpenSearchTestCase { - + private QueryGroupTaskCancellationService mockTaskCancellationService; + private ClusterService mockClusterService; + private ThreadPool mockThreadPool; + private WorkloadManagementSettings mockWorkloadManagementSettings; private ThreadPool threadPool; private WorkloadManagementTransportInterceptor sut; + private QueryGroupsStateAccessor stateAccessor; public void setUp() throws Exception { super.setUp(); + mockTaskCancellationService = mock(QueryGroupTaskCancellationService.class); + mockClusterService = mock(ClusterService.class); + mockThreadPool = mock(ThreadPool.class); + mockWorkloadManagementSettings = mock(WorkloadManagementSettings.class); threadPool = new TestThreadPool(getTestName()); - sut = new WorkloadManagementTransportInterceptor(threadPool, new QueryGroupService()); + stateAccessor = new QueryGroupsStateAccessor(); + + ClusterState state = mock(ClusterState.class); + Metadata metadata = mock(Metadata.class); + when(mockClusterService.state()).thenReturn(state); + when(state.metadata()).thenReturn(metadata); + when(metadata.queryGroups()).thenReturn(Collections.emptyMap()); + sut = new WorkloadManagementTransportInterceptor( + threadPool, + new QueryGroupService( + mockTaskCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + stateAccessor + ) + ); } public void tearDown() throws Exception { diff --git a/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java index f7a49235efc69..13e8e2c527073 100644 --- a/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/cancellation/QueryGroupTaskCancellationServiceTests.java @@ -17,12 +17,16 @@ import org.opensearch.wlm.MutableQueryGroupFragment.ResiliencyMode; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.QueryGroupsStateAccessor; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WlmMode; import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.stats.QueryGroupState; import org.opensearch.wlm.tracker.QueryGroupResourceUsageTrackerService; import org.opensearch.wlm.tracker.ResourceUsageCalculatorTrackerServiceTests.TestClock; import org.junit.Before; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -31,7 +35,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,6 +53,7 @@ public class QueryGroupTaskCancellationServiceTests extends OpenSearchTestCase { private QueryGroupTaskCancellationService taskCancellation; private WorkloadManagementSettings workloadManagementSettings; private QueryGroupResourceUsageTrackerService resourceUsageTrackerService; + private QueryGroupsStateAccessor stateAccessor; @Before public void setup() { @@ -59,12 +66,13 @@ public void setup() { when(workloadManagementSettings.getNodeLevelCpuCancellationThreshold()).thenReturn(0.9); when(workloadManagementSettings.getNodeLevelMemoryCancellationThreshold()).thenReturn(0.9); resourceUsageTrackerService = mock(QueryGroupResourceUsageTrackerService.class); + stateAccessor = mock(QueryGroupsStateAccessor.class); + when(stateAccessor.getQueryGroupState(any())).thenReturn(new QueryGroupState()); taskCancellation = new QueryGroupTaskCancellationService( workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); } @@ -138,7 +146,7 @@ public void testGetCancellableTasksFrom_returnsTasksWhenBreachingThresholdForMem activeQueryGroups.add(queryGroup1); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, cancellableTasksFrom.size()); assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); @@ -187,11 +195,10 @@ public void testGetCancellableTasksFrom_filtersQueryGroupCorrectly() { workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT, activeQueryGroups); assertEquals(0, cancellableTasksFrom.size()); } @@ -219,19 +226,19 @@ public void testCancelTasks_cancelsGivenTasks() { workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, cancellableTasksFrom.size()); assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); - taskCancellation.cancelTasks(() -> false); + when(workloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + taskCancellation.cancelTasks(() -> false, activeQueryGroups, deletedQueryGroups); assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); } @@ -281,13 +288,11 @@ public void testCancelTasks_cancelsTasksFromDeletedQueryGroups() { workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); - taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, cancellableTasksFrom.size()); assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); @@ -298,7 +303,8 @@ public void testCancelTasks_cancelsTasksFromDeletedQueryGroups() { assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); - taskCancellation.cancelTasks(() -> true); + when(workloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + taskCancellation.cancelTasks(() -> true, activeQueryGroups, deletedQueryGroups); assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); @@ -352,12 +358,11 @@ public void testCancelTasks_does_not_cancelTasksFromDeletedQueryGroups_whenNodeN workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, cancellableTasksFrom.size()); assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); @@ -368,7 +373,8 @@ public void testCancelTasks_does_not_cancelTasksFromDeletedQueryGroups_whenNodeN assertEquals(1001, cancellableTasksFromDeletedQueryGroups.get(1).getTask().getId()); when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); - taskCancellation.cancelTasks(() -> false); + when(workloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + taskCancellation.cancelTasks(() -> false, activeQueryGroups, deletedQueryGroups); assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); @@ -411,24 +417,24 @@ public void testCancelTasks_cancelsGivenTasks_WhenNodeInDuress() { workloadManagementSettings, new MaximumResourceTaskSelectionStrategy(), resourceUsageTrackerService, - activeQueryGroups, - deletedQueryGroups + stateAccessor ); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List cancellableTasksFrom = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, cancellableTasksFrom.size()); assertEquals(1234, cancellableTasksFrom.get(0).getTask().getId()); assertEquals(4321, cancellableTasksFrom.get(1).getTask().getId()); - List cancellableTasksFrom1 = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT); + List cancellableTasksFrom1 = taskCancellation.getAllCancellableTasks(ResiliencyMode.SOFT, activeQueryGroups); assertEquals(2, cancellableTasksFrom1.size()); assertEquals(5678, cancellableTasksFrom1.get(0).getTask().getId()); assertEquals(8765, cancellableTasksFrom1.get(1).getTask().getId()); when(resourceUsageTrackerService.constructQueryGroupLevelUsageViews()).thenReturn(queryGroupLevelViews); - taskCancellation.cancelTasks(() -> true); + when(workloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + taskCancellation.cancelTasks(() -> true, activeQueryGroups, deletedQueryGroups); assertTrue(cancellableTasksFrom.get(0).getTask().isCancelled()); assertTrue(cancellableTasksFrom.get(1).getTask().isCancelled()); assertTrue(cancellableTasksFrom1.get(0).getTask().isCancelled()); @@ -456,7 +462,7 @@ public void testGetAllCancellableTasks_ReturnsNoTasksWhenNotBreachingThresholds( activeQueryGroups.add(queryGroup1); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertTrue(allCancellableTasks.isEmpty()); } @@ -479,7 +485,7 @@ public void testGetAllCancellableTasks_ReturnsTasksWhenBreachingThresholds() { activeQueryGroups.add(queryGroup1); taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; - List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED); + List allCancellableTasks = taskCancellation.getAllCancellableTasks(ResiliencyMode.ENFORCED, activeQueryGroups); assertEquals(2, allCancellableTasks.size()); assertEquals(1234, allCancellableTasks.get(0).getTask().getId()); assertEquals(4321, allCancellableTasks.get(1).getTask().getId()); @@ -513,6 +519,48 @@ public void testGetCancellableTasksFrom_doesNotReturnTasksWhenQueryGroupIdNotFou assertEquals(0, cancellableTasksFrom.size()); } + public void testPruneDeletedQueryGroups() { + QueryGroup queryGroup1 = new QueryGroup( + "testQueryGroup1", + queryGroupId1, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.2)), + 1L + ); + QueryGroup queryGroup2 = new QueryGroup( + "testQueryGroup2", + queryGroupId2, + new MutableQueryGroupFragment(ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.1)), + 1L + ); + List deletedQueryGroups = new ArrayList<>(); + deletedQueryGroups.add(queryGroup1); + deletedQueryGroups.add(queryGroup2); + QueryGroupLevelResourceUsageView resourceUsageView1 = createResourceUsageViewMock(); + + List activeTasks = IntStream.range(0, 5).mapToObj(this::getRandomSearchTask).collect(Collectors.toList()); + when(resourceUsageView1.getActiveTasks()).thenReturn(activeTasks); + + QueryGroupLevelResourceUsageView resourceUsageView2 = createResourceUsageViewMock(); + when(resourceUsageView2.getActiveTasks()).thenReturn(new ArrayList<>()); + + queryGroupLevelViews.put(queryGroupId1, resourceUsageView1); + queryGroupLevelViews.put(queryGroupId2, resourceUsageView2); + + QueryGroupTaskCancellationService taskCancellation = new QueryGroupTaskCancellationService( + workloadManagementSettings, + new MaximumResourceTaskSelectionStrategy(), + resourceUsageTrackerService, + stateAccessor + ); + taskCancellation.queryGroupLevelResourceUsageViews = queryGroupLevelViews; + + taskCancellation.pruneDeletedQueryGroups(deletedQueryGroups); + + assertEquals(1, deletedQueryGroups.size()); + assertEquals(queryGroupId1, deletedQueryGroups.get(0).get_id()); + + } + private QueryGroupLevelResourceUsageView createResourceUsageViewMock() { QueryGroupLevelResourceUsageView mockView = mock(QueryGroupLevelResourceUsageView.class); when(mockView.getActiveTasks()).thenReturn(List.of(getRandomSearchTask(1234), getRandomSearchTask(4321))); diff --git a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java index 0307ff623c408..1127b50399d24 100644 --- a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java +++ b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java @@ -8,6 +8,9 @@ package org.opensearch.wlm.listeners; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.test.OpenSearchTestCase; @@ -15,12 +18,16 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.wlm.QueryGroupService; import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.QueryGroupsStateAccessor; import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.WorkloadManagementSettings; +import org.opensearch.wlm.cancellation.QueryGroupTaskCancellationService; import org.opensearch.wlm.stats.QueryGroupState; import org.opensearch.wlm.stats.QueryGroupStats; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,18 +35,24 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class QueryGroupRequestOperationListenerTests extends OpenSearchTestCase { public static final int ITERATIONS = 20; ThreadPool testThreadPool; QueryGroupService queryGroupService; - + private QueryGroupTaskCancellationService taskCancellationService; + private ClusterService mockClusterService; + private WorkloadManagementSettings mockWorkloadManagementSettings; Map queryGroupStateMap; String testQueryGroupId; QueryGroupRequestOperationListener sut; public void setUp() throws Exception { super.setUp(); + taskCancellationService = mock(QueryGroupTaskCancellationService.class); + mockClusterService = mock(ClusterService.class); + mockWorkloadManagementSettings = mock(WorkloadManagementSettings.class); queryGroupStateMap = new HashMap<>(); testQueryGroupId = "safjgagnakg-3r3fads"; testThreadPool = new TestThreadPool("RejectionTestThreadPool"); @@ -77,6 +90,21 @@ public void testValidQueryGroupRequestFailure() throws IOException { 0, 1, 0, + 0, + Map.of( + ResourceType.CPU, + new QueryGroupStats.ResourceStats(0, 0, 0), + ResourceType.MEMORY, + new QueryGroupStats.ResourceStats(0, 0, 0) + ) + ), + QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(), + new QueryGroupStats.QueryGroupStatsHolder( + 0, + 0, + 0, + 0, + 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -93,8 +121,18 @@ public void testValidQueryGroupRequestFailure() throws IOException { public void testMultiThreadedValidQueryGroupRequestFailures() { queryGroupStateMap.put(testQueryGroupId, new QueryGroupState()); - - queryGroupService = new QueryGroupService(queryGroupStateMap); + QueryGroupsStateAccessor accessor = new QueryGroupsStateAccessor(queryGroupStateMap); + setupMockedQueryGroupsFromClusterState(); + queryGroupService = new QueryGroupService( + taskCancellationService, + mockClusterService, + testThreadPool, + mockWorkloadManagementSettings, + null, + accessor, + Collections.emptySet(), + Collections.emptySet() + ); sut = new QueryGroupRequestOperationListener(queryGroupService, testThreadPool); @@ -127,6 +165,21 @@ public void testMultiThreadedValidQueryGroupRequestFailures() { 0, ITERATIONS, 0, + 0, + Map.of( + ResourceType.CPU, + new QueryGroupStats.ResourceStats(0, 0, 0), + ResourceType.MEMORY, + new QueryGroupStats.ResourceStats(0, 0, 0) + ) + ), + QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(), + new QueryGroupStats.QueryGroupStatsHolder( + 0, + 0, + 0, + 0, + 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -149,6 +202,21 @@ public void testInvalidQueryGroupFailure() throws IOException { 0, 0, 0, + 0, + Map.of( + ResourceType.CPU, + new QueryGroupStats.ResourceStats(0, 0, 0), + ResourceType.MEMORY, + new QueryGroupStats.ResourceStats(0, 0, 0) + ) + ), + QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(), + new QueryGroupStats.QueryGroupStatsHolder( + 0, + 0, + 1, + 0, + 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -169,12 +237,23 @@ private void assertSuccess( QueryGroupStats expectedStats, String threadContextQG_Id ) { - + QueryGroupsStateAccessor stateAccessor = new QueryGroupsStateAccessor(queryGroupStateMap); try (ThreadContext.StoredContext currentContext = testThreadPool.getThreadContext().stashContext()) { testThreadPool.getThreadContext().putHeader(QueryGroupTask.QUERY_GROUP_ID_HEADER, threadContextQG_Id); queryGroupStateMap.put(testQueryGroupId, new QueryGroupState()); - queryGroupService = new QueryGroupService(queryGroupStateMap); + setupMockedQueryGroupsFromClusterState(); + + queryGroupService = new QueryGroupService( + taskCancellationService, + mockClusterService, + testThreadPool, + mockWorkloadManagementSettings, + null, + stateAccessor, + Collections.emptySet(), + Collections.emptySet() + ); sut = new QueryGroupRequestOperationListener(queryGroupService, testThreadPool); sut.onRequestFailure(null, null); @@ -184,4 +263,12 @@ private void assertSuccess( } } + + private void setupMockedQueryGroupsFromClusterState() { + ClusterState state = mock(ClusterState.class); + Metadata metadata = mock(Metadata.class); + when(mockClusterService.state()).thenReturn(state); + when(state.metadata()).thenReturn(metadata); + when(metadata.queryGroups()).thenReturn(Collections.emptyMap()); + } } diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java index 576eec7be1888..566c4261d6878 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -23,7 +23,13 @@ public void testRandomQueryGroupsStateUpdates() { for (int i = 0; i < 25; i++) { if (i % 5 == 0) { - updaterThreads.add(new Thread(() -> queryGroupState.completions.inc())); + updaterThreads.add(new Thread(() -> { + if (randomBoolean()) { + queryGroupState.completions.inc(); + } else { + queryGroupState.shardCompletions.inc(); + } + })); } else if (i % 5 == 1) { updaterThreads.add(new Thread(() -> { queryGroupState.totalRejections.inc(); @@ -57,7 +63,7 @@ public void testRandomQueryGroupsStateUpdates() { } }); - assertEquals(5, queryGroupState.getCompletions()); + assertEquals(5, queryGroupState.getCompletions() + queryGroupState.getShardCompletions()); assertEquals(5, queryGroupState.getTotalRejections()); final long sumOfRejectionsDueToResourceTypes = queryGroupState.getResourceState().get(ResourceType.CPU).rejections.count() diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index 661c3a7beae40..ac6d19580dacb 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -28,9 +28,10 @@ public void testToXContent() throws IOException { queryGroupId, new QueryGroupStats.QueryGroupStatsHolder( 123456789, + 13, 2, 0, - 13, + 1213718, Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(0.3, 13, 2)) ) ); @@ -40,7 +41,7 @@ public void testToXContent() throws IOException { queryGroupStats.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); assertEquals( - "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"rejections\":2,\"failures\":0,\"total_cancellations\":13,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", + "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"shard_completions\":1213718,\"rejections\":13,\"failures\":2,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", builder.toString() ); } @@ -60,6 +61,7 @@ protected QueryGroupStats createTestInstance() { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats( From 266bdd30054f5d9110bf136c7b4f8f4564f9d767 Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:25:09 -0700 Subject: [PATCH 195/357] [Tiered Caching] Segmented cache changes (#16047) * Segmented cache changes for TieredCache Signed-off-by: Sagar Upadhyaya * Adding change log Signed-off-by: Sagar Upadhyaya * Allow segment number to be power of two Signed-off-by: Sagar Upadhyaya * Moving common tiered cache IT methods to a common base class Signed-off-by: Sagar Upadhyaya * Adding disk took time IT test with multiple segment Signed-off-by: Sagar Upadhyaya * Correcting changelog Signed-off-by: Sagar Upadhyaya * Addressing comments Signed-off-by: Sagar Upadhyaya * Fixing invalid segment count variable name Signed-off-by: Sagar Upadhyaya * Introducing new settings for size for respective cache tier Signed-off-by: Sagar Upadhyaya * Changing the default segmentCount logic Signed-off-by: Sagar Upadhyaya * Fixing missing java doc issue Signed-off-by: Sagar Upadhyaya --------- Signed-off-by: Sagar Upadhyaya Signed-off-by: Sagar <99425694+sgup432@users.noreply.github.com> --- CHANGELOG.md | 1 + .../tier/TieredSpilloverCacheBaseIT.java | 57 + .../common/tier/TieredSpilloverCacheIT.java | 169 ++- .../tier/TieredSpilloverCacheStatsIT.java | 100 +- .../common/tier/TieredSpilloverCache.java | 884 +++++++++----- .../tier/TieredSpilloverCachePlugin.java | 9 + .../tier/TieredSpilloverCacheSettings.java | 69 ++ .../cache/common/tier/MockDiskCache.java | 27 +- .../tier/TieredSpilloverCacheTests.java | 1087 +++++++++++++---- .../cache/EhcacheDiskCacheSettings.java | 2 +- .../cache/store/disk/EhcacheDiskCache.java | 55 +- .../store/disk/EhCacheDiskCacheTests.java | 68 ++ .../org/opensearch/common/cache/Cache.java | 37 +- .../opensearch/common/cache/CacheBuilder.java | 15 +- .../common/cache/settings/CacheSettings.java | 13 + .../cache/store/OpenSearchOnHeapCache.java | 27 +- .../cache/store/builders/ICacheBuilder.java | 11 + .../cache/store/config/CacheConfig.java | 39 + .../indices/IndicesRequestCache.java | 5 +- .../opensearch/indices/IndicesService.java | 2 +- .../opensearch/common/cache/CacheTests.java | 64 +- .../store/OpenSearchOnHeapCacheTests.java | 31 + .../indices/IndicesRequestCacheTests.java | 63 +- 23 files changed, 2170 insertions(+), 665 deletions(-) create mode 100644 modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8cdd882bbb9..06ac7740eea6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) +- [Tiered Caching] Segmented cache changes ([#16047](https://github.com/opensearch-project/OpenSearch/pull/16047)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) diff --git a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java new file mode 100644 index 0000000000000..01371ca8eeefb --- /dev/null +++ b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache.common.tier; + +import org.opensearch.common.cache.CacheType; +import org.opensearch.common.cache.settings.CacheSettings; +import org.opensearch.common.cache.store.OpenSearchOnHeapCache; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.OpenSearchIntegTestCase; + +public class TieredSpilloverCacheBaseIT extends OpenSearchIntegTestCase { + + public Settings defaultSettings(String onHeapCacheSizeInBytesOrPercentage, int numberOfSegments) { + return Settings.builder() + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + MockDiskCache.MockDiskCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + numberOfSegments + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSizeInBytesOrPercentage + ) + .build(); + } + + public int getNumberOfSegments() { + return randomFrom(1, 2, 4, 8, 16, 64, 128, 256); + } +} diff --git a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheIT.java b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheIT.java index 02be0990eb136..d58e36c036510 100644 --- a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheIT.java +++ b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheIT.java @@ -22,12 +22,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.ICache; -import org.opensearch.common.cache.settings.CacheSettings; -import org.opensearch.common.cache.store.OpenSearchOnHeapCache; -import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesRequestCache; @@ -43,13 +39,15 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY; +import static org.opensearch.common.cache.settings.CacheSettings.INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE; import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -58,43 +56,15 @@ import static org.hamcrest.Matchers.greaterThan; @OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST) -public class TieredSpilloverCacheIT extends OpenSearchIntegTestCase { +public class TieredSpilloverCacheIT extends TieredSpilloverCacheBaseIT { @Override protected Collection> nodePlugins() { return Arrays.asList(TieredSpilloverCachePlugin.class, MockDiskCachePlugin.class); } - static Settings defaultSettings(String onHeapCacheSizeInBytesOrPercentage) { - return Settings.builder() - .put(FeatureFlags.PLUGGABLE_CACHE, "true") - .put( - CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), - TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME - ) - .put( - TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( - CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() - ).getKey(), - OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME - ) - .put( - TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( - CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() - ).getKey(), - MockDiskCache.MockDiskCacheFactory.NAME - ) - .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), - onHeapCacheSizeInBytesOrPercentage - ) - .build(); - } - public void testPluginsAreInstalled() { - internalCluster().startNode(Settings.builder().put(defaultSettings("1%")).build()); + internalCluster().startNode(Settings.builder().put(defaultSettings("1%", getNumberOfSegments())).build()); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(); nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName()); NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); @@ -111,7 +81,8 @@ public void testPluginsAreInstalled() { } public void testSanityChecksWithIndicesRequestCache() throws InterruptedException { - internalCluster().startNodes(3, Settings.builder().put(defaultSettings("1%")).build()); + int numberOfSegments = getNumberOfSegments(); + internalCluster().startNodes(3, Settings.builder().put(defaultSettings("1%", numberOfSegments)).build()); Client client = client(); assertAcked( client.admin() @@ -147,9 +118,97 @@ public void testSanityChecksWithIndicesRequestCache() throws InterruptedExceptio ); } + public void testWithDynamicTookTimePolicyWithMultiSegments() throws Exception { + int numberOfSegments = getNumberOfSegments(); + int onHeapCacheSizePerSegmentInBytes = 800; // Per cache entry below is around ~700 bytes, so keeping this + // just a bit higher so that each segment can atleast hold 1 entry. + int onHeapCacheSizeInBytes = onHeapCacheSizePerSegmentInBytes * numberOfSegments; + internalCluster().startNode(Settings.builder().put(defaultSettings(onHeapCacheSizeInBytes + "b", numberOfSegments)).build()); + Client client = client(); + assertAcked( + client.admin() + .indices() + .prepareCreate("index") + .setMapping("k", "type=keyword") + .setSettings( + Settings.builder() + .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", -1) + ) + .get() + ); + // Set a very high value for took time policy so that no items evicted from onHeap cache are spilled + // to disk. And then hit requests so that few items are cached into cache. + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings( + Settings.builder() + .put( + TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), + new TimeValue(100, TimeUnit.SECONDS) + ) + .build() + ); + assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get()); + int numberOfIndexedItems = numberOfSegments + 1; // Best case if all keys are distributed among different + // segment, atleast one of the segment will have 2 entries and we will see evictions. + for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) { + indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator)); + } + ensureSearchable("index"); + refreshAndWaitForReplication(); + // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache + ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get(); + OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); + long perQuerySizeInCacheInBytes = -1; + for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) { + SearchResponse resp = client.prepareSearch("index") + .setRequestCache(true) + .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator)) + .get(); + if (perQuerySizeInCacheInBytes == -1) { + RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index"); + perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes(); + } + assertSearchResponse(resp); + } + RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index"); + // Considering disk cache won't be used due to took time policy having a high value, we expect overall cache + // size to be less than or equal to onHeapCache size. + assertTrue(requestCacheStats.getMemorySizeInBytes() <= onHeapCacheSizeInBytes); + assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount()); + // We should atleast one eviction considering disk cache isn't able to hold anything due to policy. + assertTrue(requestCacheStats.getEvictions() > 0); + assertEquals(0, requestCacheStats.getHitCount()); + long lastEvictionSeen = requestCacheStats.getEvictions(); + + // Decrease took time policy to zero so that disk cache also comes into play. Now we should be able + // to cache all entries. + updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings( + Settings.builder() + .put( + TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), + new TimeValue(0, TimeUnit.MILLISECONDS) + ) + .build() + ); + assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get()); + for (int iterator = 0; iterator < numberOfIndexedItems * 2; iterator++) { + SearchResponse resp = client.prepareSearch("index") + .setRequestCache(true) + .setQuery(QueryBuilders.termQuery(UUID.randomUUID().toString(), UUID.randomUUID().toString())) + .get(); + assertSearchResponse(resp); + } + + requestCacheStats = getRequestCacheStats(client, "index"); + // We shouldn't see any new evictions now. + assertEquals(lastEvictionSeen, requestCacheStats.getEvictions()); + } + public void testWithDynamicTookTimePolicy() throws Exception { int onHeapCacheSizeInBytes = 2000; - internalCluster().startNode(Settings.builder().put(defaultSettings(onHeapCacheSizeInBytes + "b")).build()); + internalCluster().startNode(Settings.builder().put(defaultSettings(onHeapCacheSizeInBytes + "b", 1)).build()); Client client = client(); assertAcked( client.admin() @@ -271,9 +330,10 @@ public void testWithDynamicTookTimePolicy() throws Exception { public void testInvalidationWithIndicesRequestCache() throws Exception { int onHeapCacheSizeInBytes = 2000; + int numberOfSegments = getNumberOfSegments(); internalCluster().startNode( Settings.builder() - .put(defaultSettings(onHeapCacheSizeInBytes + "b")) + .put(defaultSettings(onHeapCacheSizeInBytes + "b", numberOfSegments)) .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1)) .build() ); @@ -354,10 +414,11 @@ public void testInvalidationWithIndicesRequestCache() throws Exception { } public void testWithExplicitCacheClear() throws Exception { + int numberOfSegments = getNumberOfSegments(); int onHeapCacheSizeInBytes = 2000; internalCluster().startNode( Settings.builder() - .put(defaultSettings(onHeapCacheSizeInBytes + "b")) + .put(defaultSettings(onHeapCacheSizeInBytes + "b", numberOfSegments)) .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1)) .build() ); @@ -426,10 +487,13 @@ public void testWithExplicitCacheClear() throws Exception { } public void testWithDynamicDiskCacheSetting() throws Exception { - int onHeapCacheSizeInBytes = 10; // Keep it low so that all items are cached onto disk. + int numberOfSegments = getNumberOfSegments(); + int onHeapCacheSizeInBytes = randomIntBetween(numberOfSegments + 1, numberOfSegments * 2); // Keep it low so + // that all items are + // cached onto disk. internalCluster().startNode( Settings.builder() - .put(defaultSettings(onHeapCacheSizeInBytes + "b")) + .put(defaultSettings(onHeapCacheSizeInBytes + "b", numberOfSegments)) .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1)) .build() ); @@ -540,6 +604,27 @@ public void testWithDynamicDiskCacheSetting() throws Exception { assertEquals(0, lastKnownHitCount - requestCacheStats.getHitCount()); } + public void testWithInvalidSegmentNumberSetting() throws Exception { + int numberOfSegments = getNumberOfSegments(); + int onHeapCacheSizeInBytes = randomIntBetween(numberOfSegments + 1, numberOfSegments * 2); // Keep it low so + // that all items are + // cached onto disk. + assertThrows( + String.format( + Locale.ROOT, + INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE, + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ), + IllegalArgumentException.class, + () -> internalCluster().startNode( + Settings.builder() + .put(defaultSettings(onHeapCacheSizeInBytes + "b", 300)) + .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1)) + .build() + ) + ); + } + private RequestCacheStats getRequestCacheStats(Client client, String indexName) { return client.admin().indices().prepareStats(indexName).setRequestCache(true).get().getTotal().getRequestCache(); } @@ -550,7 +635,7 @@ public MockDiskCachePlugin() {} @Override public Map getCacheFactoryMap() { - return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 1000, false)); + return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 10000, false, 1)); } @Override diff --git a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java index 783b6083e9226..fe6bd7050a8f3 100644 --- a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java +++ b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java @@ -45,16 +45,16 @@ // Use a single data node to simplify accessing cache stats across different shards. @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class TieredSpilloverCacheStatsIT extends OpenSearchIntegTestCase { +public class TieredSpilloverCacheStatsIT extends TieredSpilloverCacheBaseIT { @Override protected Collection> nodePlugins() { return Arrays.asList(TieredSpilloverCachePlugin.class, TieredSpilloverCacheIT.MockDiskCachePlugin.class); } - private final String HEAP_CACHE_SIZE_STRING = "10000B"; - private final int HEAP_CACHE_SIZE = 10_000; - private final String index1Name = "index1"; - private final String index2Name = "index2"; + private static final String HEAP_CACHE_SIZE_STRING = "10000B"; + private static final int HEAP_CACHE_SIZE = 10_000; + private static final String index1Name = "index1"; + private static final String index2Name = "index2"; /** * Test aggregating by indices @@ -63,7 +63,7 @@ public void testIndicesLevelAggregation() throws Exception { internalCluster().startNodes( 1, Settings.builder() - .put(TieredSpilloverCacheIT.defaultSettings(HEAP_CACHE_SIZE_STRING)) + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, 1)) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(0, TimeUnit.SECONDS) @@ -116,7 +116,7 @@ public void testIndicesAndTierLevelAggregation() throws Exception { internalCluster().startNodes( 1, Settings.builder() - .put(TieredSpilloverCacheIT.defaultSettings(HEAP_CACHE_SIZE_STRING)) + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, 1)) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(0, TimeUnit.SECONDS) @@ -196,7 +196,7 @@ public void testTierLevelAggregation() throws Exception { internalCluster().startNodes( 1, Settings.builder() - .put(TieredSpilloverCacheIT.defaultSettings(HEAP_CACHE_SIZE_STRING)) + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, 1)) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(0, TimeUnit.SECONDS) @@ -205,7 +205,6 @@ public void testTierLevelAggregation() throws Exception { ); Client client = client(); Map values = setupCacheForAggregationTests(client); - // Get values for tiers alone and check they add correctly across indices ImmutableCacheStatsHolder tiersOnlyStatsHolder = getNodeCacheStatsResult(client, List.of(TIER_DIMENSION_NAME)); ImmutableCacheStats totalHeapExpectedStats = returnNullIfAllZero( @@ -238,7 +237,7 @@ public void testInvalidLevelsAreIgnored() throws Exception { internalCluster().startNodes( 1, Settings.builder() - .put(TieredSpilloverCacheIT.defaultSettings(HEAP_CACHE_SIZE_STRING)) + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, getNumberOfSegments())) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(0, TimeUnit.SECONDS) @@ -289,7 +288,7 @@ public void testStatsMatchOldApi() throws Exception { internalCluster().startNodes( 1, Settings.builder() - .put(TieredSpilloverCacheIT.defaultSettings(HEAP_CACHE_SIZE_STRING)) + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, getNumberOfSegments())) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(0, TimeUnit.SECONDS) @@ -342,6 +341,82 @@ public void testStatsMatchOldApi() throws Exception { assertEquals(oldAPIStats.getMemorySizeInBytes(), totalStats.getSizeInBytes()); } + public void testStatsWithMultipleSegments() throws Exception { + int numberOfSegments = randomFrom(2, 4, 8, 16, 64); + int singleSearchSizeApproxUpperBound = 700; // We know this from other tests and manually verifying + int heap_cache_size_per_segment = singleSearchSizeApproxUpperBound * numberOfSegments; // Worst case if all + // keys land up in same segment, it would still be able to accommodate. + internalCluster().startNodes( + 1, + Settings.builder() + .put(defaultSettings(heap_cache_size_per_segment * numberOfSegments + "B", numberOfSegments)) + .put( + TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), + new TimeValue(0, TimeUnit.SECONDS) + ) + .build() + ); + Client client = client(); + startIndex(client, index1Name); + // First search one time to calculate item size + searchIndex(client, index1Name, 0); + // get total stats + long singleSearchSize = getTotalStats(client).getSizeInBytes(); + // Now try to hit queries same as number of segments. All these should be able to reside inside onHeap cache. + for (int i = 1; i < numberOfSegments; i++) { + searchIndex(client, index1Name, i); + } + ImmutableCacheStatsHolder allLevelsStatsHolder = getNodeCacheStatsResult( + client, + List.of(IndicesRequestCache.INDEX_DIMENSION_NAME, TIER_DIMENSION_NAME) + ); + ImmutableCacheStats index1OnHeapExpectedStats = returnNullIfAllZero( + new ImmutableCacheStats(0, numberOfSegments, 0, singleSearchSize * numberOfSegments, numberOfSegments) + ); + assertEquals( + index1OnHeapExpectedStats, + allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_ON_HEAP)) + ); + ImmutableCacheStats index1DiskCacheExpectedStats = returnNullIfAllZero(new ImmutableCacheStats(0, numberOfSegments, 0, 0, 0)); + assertEquals( + index1DiskCacheExpectedStats, + allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_DISK)) + ); + + // Now fire same queries to get some hits + for (int i = 0; i < numberOfSegments; i++) { + searchIndex(client, index1Name, i); + } + allLevelsStatsHolder = getNodeCacheStatsResult(client, List.of(IndicesRequestCache.INDEX_DIMENSION_NAME, TIER_DIMENSION_NAME)); + index1OnHeapExpectedStats = returnNullIfAllZero( + new ImmutableCacheStats(numberOfSegments, numberOfSegments, 0, singleSearchSize * numberOfSegments, numberOfSegments) + ); + assertEquals( + index1OnHeapExpectedStats, + allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_ON_HEAP)) + ); + + // Now try to evict from onheap cache by adding numberOfSegments ^ 2 which will guarantee this. + for (int i = numberOfSegments; i < numberOfSegments + numberOfSegments * numberOfSegments; i++) { + searchIndex(client, index1Name, i); + } + allLevelsStatsHolder = getNodeCacheStatsResult(client, List.of(IndicesRequestCache.INDEX_DIMENSION_NAME, TIER_DIMENSION_NAME)); + ImmutableCacheStats onHeapCacheStat = allLevelsStatsHolder.getStatsForDimensionValues( + List.of(index1Name, TIER_DIMENSION_VALUE_ON_HEAP) + ); + // Jut verifying evictions happened as can't fetch the exact number considering we don't have a way to get + // segment number for queries. + assertTrue(onHeapCacheStat.getEvictions() > 0); + ImmutableCacheStats diskCacheStat = allLevelsStatsHolder.getStatsForDimensionValues(List.of(index1Name, TIER_DIMENSION_VALUE_DISK)); + + // Similarly verify items are present on disk cache now + assertEquals(onHeapCacheStat.getEvictions(), diskCacheStat.getItems()); + assertTrue(diskCacheStat.getSizeInBytes() > 0); + assertTrue(diskCacheStat.getMisses() > 0); + assertTrue(diskCacheStat.getHits() == 0); + assertTrue(diskCacheStat.getEvictions() == 0); + } + private void startIndex(Client client, String indexName) throws InterruptedException { assertAcked( client.admin() @@ -373,6 +448,7 @@ private Map setupCacheForAggregationTests(Client client) throws searchIndex(client, index1Name, 0); // get total stats long singleSearchSize = getTotalStats(client).getSizeInBytes(); + int itemsOnHeapAfterTest = HEAP_CACHE_SIZE / (int) singleSearchSize; // As the heap tier evicts, the items on it after the test will // be the same as its max capacity int itemsOnDiskAfterTest = 1 + randomInt(30); // The first one we search (to get the size) always goes to disk @@ -416,7 +492,6 @@ private Map setupCacheForAggregationTests(Client client) throws for (int i = itemsOnDiskAfterTest + itemsOnHeapIndex1AfterTest; i < itemsOnDiskAfterTest + itemsOnHeapAfterTest; i++) { searchIndex(client, index2Name, i); } - // Get some hits on all combinations of indices and tiers for (int i = itemsOnDiskAfterTest; i < itemsOnDiskAfterTest + hitsOnHeapIndex1; i++) { // heap hits for index 1 @@ -499,6 +574,7 @@ private static ImmutableCacheStatsHolder getNodeCacheStatsResult(Client client, .addMetric(NodesStatsRequest.Metric.CACHE_STATS.metricName()) .setIndices(statsFlags) .get(); + // Can always get the first data node as there's only one in this test suite assertEquals(1, nodeStatsResponse.getNodes().size()); NodeCacheStats ncs = nodeStatsResponse.getNodes().get(0).getNodeCacheStats(); diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java index f69c56808b2a1..ab5335ca0ca66 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java @@ -34,6 +34,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; @@ -49,8 +50,13 @@ import java.util.function.ToLongBiFunction; import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_SIZE; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP; +import static org.opensearch.common.cache.settings.CacheSettings.INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE; +import static org.opensearch.common.cache.settings.CacheSettings.VALID_SEGMENT_COUNT_VALUES; /** * This cache spillover the evicted items from heap tier to disk tier. All the new items are first cached on heap @@ -69,29 +75,16 @@ public class TieredSpilloverCache implements ICache { private static final List SPILLOVER_REMOVAL_REASONS = List.of(RemovalReason.EVICTED, RemovalReason.CAPACITY); private static final Logger logger = LogManager.getLogger(TieredSpilloverCache.class); - private final ICache diskCache; - private final ICache onHeapCache; - - // Removal listeners for the individual tiers - private final RemovalListener, V> onDiskRemovalListener; - private final RemovalListener, V> onHeapRemovalListener; - - // Removal listener from the spillover cache as a whole - private final RemovalListener, V> removalListener; + static final String ZERO_SEGMENT_COUNT_EXCEPTION_MESSAGE = "Segment count cannot be less than one for tiered cache"; // In future we want to just read the stats from the individual tiers' statsHolder objects, but this isn't // possible right now because of the way computeIfAbsent is implemented. private final TieredSpilloverCacheStatsHolder statsHolder; - private ToLongBiFunction, V> weigher; private final List dimensionNames; - ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock()); - ReleasableLock writeLock = new ReleasableLock(readWriteLock.writeLock()); - /** - * Maintains caching tiers in ascending order of cache latency. - */ - private final Map, TierInfo> caches; - private final List> policies; + + private final int numberOfSegments; + + final TieredSpilloverCacheSegment[] tieredSpilloverCacheSegments; /** * This map is used to handle concurrent requests for same key in computeIfAbsent() to ensure we load the value @@ -99,224 +92,525 @@ public class TieredSpilloverCache implements ICache { */ Map, CompletableFuture, V>>> completableFutureMap = new ConcurrentHashMap<>(); + @SuppressWarnings({ "unchecked", "rawtypes" }) TieredSpilloverCache(Builder builder) { - Objects.requireNonNull(builder.onHeapCacheFactory, "onHeap cache builder can't be null"); - Objects.requireNonNull(builder.diskCacheFactory, "disk cache builder can't be null"); Objects.requireNonNull(builder.cacheConfig, "cache config can't be null"); - Objects.requireNonNull(builder.cacheConfig.getClusterSettings(), "cluster settings can't be null"); - this.removalListener = Objects.requireNonNull(builder.removalListener, "Removal listener can't be null"); - - this.onHeapRemovalListener = new HeapTierRemovalListener(this); - this.onDiskRemovalListener = new DiskTierRemovalListener(this); - this.weigher = Objects.requireNonNull(builder.cacheConfig.getWeigher(), "Weigher can't be null"); - - this.onHeapCache = builder.onHeapCacheFactory.create( - new CacheConfig.Builder().setRemovalListener(onHeapRemovalListener) - .setKeyType(builder.cacheConfig.getKeyType()) - .setValueType(builder.cacheConfig.getValueType()) - .setSettings(builder.cacheConfig.getSettings()) - .setWeigher(builder.cacheConfig.getWeigher()) - .setDimensionNames(builder.cacheConfig.getDimensionNames()) - .setMaxSizeInBytes(builder.cacheConfig.getMaxSizeInBytes()) - .setExpireAfterAccess(builder.cacheConfig.getExpireAfterAccess()) - .setClusterSettings(builder.cacheConfig.getClusterSettings()) - .setStatsTrackingEnabled(false) - .build(), - builder.cacheType, - builder.cacheFactories - - ); - this.diskCache = builder.diskCacheFactory.create( - new CacheConfig.Builder().setRemovalListener(onDiskRemovalListener) - .setKeyType(builder.cacheConfig.getKeyType()) - .setValueType(builder.cacheConfig.getValueType()) - .setSettings(builder.cacheConfig.getSettings()) - .setWeigher(builder.cacheConfig.getWeigher()) - .setKeySerializer(builder.cacheConfig.getKeySerializer()) - .setValueSerializer(builder.cacheConfig.getValueSerializer()) - .setDimensionNames(builder.cacheConfig.getDimensionNames()) - .setStatsTrackingEnabled(false) - .build(), - builder.cacheType, - builder.cacheFactories - ); + Objects.requireNonNull(builder.cacheConfig.getSettings(), "settings can't be null"); + if (builder.numberOfSegments <= 0) { + throw new IllegalArgumentException(ZERO_SEGMENT_COUNT_EXCEPTION_MESSAGE); + } + this.numberOfSegments = builder.numberOfSegments; Boolean isDiskCacheEnabled = DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType).get(builder.cacheConfig.getSettings()); - LinkedHashMap, TierInfo> cacheListMap = new LinkedHashMap<>(); - cacheListMap.put(onHeapCache, new TierInfo(true, TIER_DIMENSION_VALUE_ON_HEAP)); - cacheListMap.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK)); - this.caches = Collections.synchronizedMap(cacheListMap); - this.dimensionNames = builder.cacheConfig.getDimensionNames(); // Pass "tier" as the innermost dimension name, in addition to whatever dimensions are specified for the cache as a whole this.statsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, isDiskCacheEnabled); - this.policies = builder.policies; // Will never be null; builder initializes it to an empty list + long onHeapCachePerSegmentSizeInBytes = builder.onHeapCacheSizeInBytes / this.numberOfSegments; + long diskCachePerSegmentSizeInBytes = builder.diskCacheSizeInBytes / this.numberOfSegments; + if (onHeapCachePerSegmentSizeInBytes <= 0) { + throw new IllegalArgumentException("Per segment size for onHeap cache within Tiered cache should be " + "greater than 0"); + } + if (diskCachePerSegmentSizeInBytes <= 0) { + throw new IllegalArgumentException("Per segment size for disk cache within Tiered cache should be " + "greater than 0"); + } + this.tieredSpilloverCacheSegments = new TieredSpilloverCacheSegment[this.numberOfSegments]; + for (int i = 0; i < numberOfSegments; i++) { + tieredSpilloverCacheSegments[i] = new TieredSpilloverCacheSegment( + builder, + statsHolder, + i + 1, + this.numberOfSegments, + onHeapCachePerSegmentSizeInBytes, + diskCachePerSegmentSizeInBytes + ); + } builder.cacheConfig.getClusterSettings() .addSettingsUpdateConsumer(DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType), this::enableDisableDiskCache); } - // Package private for testing - ICache getOnHeapCache() { - return onHeapCache; - } + static class TieredSpilloverCacheSegment implements ICache { - // Package private for testing - ICache getDiskCache() { - return diskCache; - } + private final ICache diskCache; + private final ICache onHeapCache; - // Package private for testing. - void enableDisableDiskCache(Boolean isDiskCacheEnabled) { - // When disk cache is disabled, we are not clearing up the disk cache entries yet as that should be part of - // separate cache/clear API. - this.caches.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK)); - this.statsHolder.setDiskCacheEnabled(isDiskCacheEnabled); - } + // Removal listeners for the individual tiers + private final RemovalListener, V> onDiskRemovalListener; + private final RemovalListener, V> onHeapRemovalListener; - @Override - public V get(ICacheKey key) { - Tuple cacheValueTuple = getValueFromTieredCache(true).apply(key); - if (cacheValueTuple == null) { - return null; + // Removal listener from the spillover cache as a whole + private final RemovalListener, V> removalListener; + + private ToLongBiFunction, V> weigher; + ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock()); + ReleasableLock writeLock = new ReleasableLock(readWriteLock.writeLock()); + + private final Map, TierInfo> caches; + + private final List> policies; + + private final TieredSpilloverCacheStatsHolder statsHolder; + + /** + * This map is used to handle concurrent requests for same key in computeIfAbsent() to ensure we load the value + * only once. + */ + Map, CompletableFuture, V>>> completableFutureMap = new ConcurrentHashMap<>(); + + TieredSpilloverCacheSegment( + Builder builder, + TieredSpilloverCacheStatsHolder statsHolder, + int segmentNumber, + int numberOfSegments, + long onHeapCacheSizeInBytes, + long diskCacheSizeInBytes + ) { + Objects.requireNonNull(builder.onHeapCacheFactory, "onHeap cache builder can't be null"); + Objects.requireNonNull(builder.diskCacheFactory, "disk cache builder can't be null"); + Objects.requireNonNull(builder.cacheConfig, "cache config can't be null"); + Objects.requireNonNull(builder.cacheConfig.getClusterSettings(), "cluster settings can't be null"); + Objects.requireNonNull(builder.cacheConfig.getStoragePath(), "Storage path shouldn't be null"); + this.removalListener = Objects.requireNonNull(builder.removalListener, "Removal listener can't be null"); + this.statsHolder = statsHolder; + + this.onHeapRemovalListener = new HeapTierRemovalListener(this); + this.onDiskRemovalListener = new DiskTierRemovalListener(this); + this.weigher = Objects.requireNonNull(builder.cacheConfig.getWeigher(), "Weigher can't be null"); + this.onHeapCache = builder.onHeapCacheFactory.create( + new CacheConfig.Builder().setRemovalListener(onHeapRemovalListener) + .setKeyType(builder.cacheConfig.getKeyType()) + .setValueType(builder.cacheConfig.getValueType()) + .setSettings(builder.cacheConfig.getSettings()) + .setWeigher(builder.cacheConfig.getWeigher()) + .setDimensionNames(builder.cacheConfig.getDimensionNames()) + .setMaxSizeInBytes(onHeapCacheSizeInBytes) + .setExpireAfterAccess(builder.cacheConfig.getExpireAfterAccess()) + .setClusterSettings(builder.cacheConfig.getClusterSettings()) + .setSegmentCount(1) // We don't need to make underlying caches multi-segmented + .setStatsTrackingEnabled(false) + .setCacheAlias("tiered_on_heap#" + segmentNumber) + .build(), + builder.cacheType, + builder.cacheFactories + + ); + this.diskCache = builder.diskCacheFactory.create( + new CacheConfig.Builder().setRemovalListener(onDiskRemovalListener) + .setKeyType(builder.cacheConfig.getKeyType()) + .setValueType(builder.cacheConfig.getValueType()) + .setSettings(builder.cacheConfig.getSettings()) + .setWeigher(builder.cacheConfig.getWeigher()) + .setKeySerializer(builder.cacheConfig.getKeySerializer()) + .setValueSerializer(builder.cacheConfig.getValueSerializer()) + .setDimensionNames(builder.cacheConfig.getDimensionNames()) + .setSegmentCount(1) // We don't need to make underlying caches multi-segmented + .setStatsTrackingEnabled(false) + .setMaxSizeInBytes(diskCacheSizeInBytes) + .setStoragePath(builder.cacheConfig.getStoragePath() + "/" + segmentNumber) + .setCacheAlias("tiered_disk_cache#" + segmentNumber) + .build(), + builder.cacheType, + builder.cacheFactories + ); + + Boolean isDiskCacheEnabled = DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType).get(builder.cacheConfig.getSettings()); + LinkedHashMap, TierInfo> cacheListMap = new LinkedHashMap<>(); + cacheListMap.put(onHeapCache, new TierInfo(true, TIER_DIMENSION_VALUE_ON_HEAP)); + cacheListMap.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK)); + this.caches = Collections.synchronizedMap(cacheListMap); + this.policies = builder.policies; // Will never be null; builder initializes it to an empty list } - return cacheValueTuple.v1(); - } - @Override - public void put(ICacheKey key, V value) { - try (ReleasableLock ignore = writeLock.acquire()) { - onHeapCache.put(key, value); - updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value); + // Package private for testing + ICache getOnHeapCache() { + return onHeapCache; } - } - @Override - public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> loader) throws Exception { - // Don't capture stats in the initial getValueFromTieredCache(). If we have concurrent requests for the same key, - // and it only has to be loaded one time, we should report one miss and the rest hits. But, if we do stats in - // getValueFromTieredCache(), - // we will see all misses. Instead, handle stats in computeIfAbsent(). - Tuple cacheValueTuple; - CompletableFuture, V>> future = null; - try (ReleasableLock ignore = readLock.acquire()) { - cacheValueTuple = getValueFromTieredCache(false).apply(key); + // Package private for testing + ICache getDiskCache() { + return diskCache; + } + + void enableDisableDiskCache(Boolean isDiskCacheEnabled) { + // When disk cache is disabled, we are not clearing up the disk cache entries yet as that should be part of + // separate cache/clear API. + this.caches.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK)); + this.statsHolder.setDiskCacheEnabled(isDiskCacheEnabled); + } + + @Override + public V get(ICacheKey key) { + Tuple cacheValueTuple = getValueFromTieredCache(true).apply(key); if (cacheValueTuple == null) { - // Only one of the threads will succeed putting a future into map for the same key. - // Rest will fetch existing future and wait on that to complete. - future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>()); + return null; } + return cacheValueTuple.v1(); } - List heapDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_ON_HEAP); - List diskDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_DISK); - - if (cacheValueTuple == null) { - // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside. - // This is needed as there can be many requests for the same key at the same time and we only want to load - // the value once. - V value = compute(key, loader, future); - // Handle stats - if (loader.isLoaded()) { - // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache - // if present + + @Override + public void put(ICacheKey key, V value) { + // First check in case the key is already present in either of tiers. + Tuple cacheValueTuple = getValueFromTieredCache(true).apply(key); + if (cacheValueTuple == null) { + // In case it is not present in any tier, put it inside onHeap cache by default. + try (ReleasableLock ignore = writeLock.acquire()) { + onHeapCache.put(key, value); + } updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value); - statsHolder.incrementMisses(heapDimensionValues); - if (caches.get(diskCache).isEnabled()) { - statsHolder.incrementMisses(diskDimensionValues); + } else { + // Put it inside desired tier. + try (ReleasableLock ignore = writeLock.acquire()) { + for (Map.Entry, TierInfo> entry : this.caches.entrySet()) { + if (cacheValueTuple.v2().equals(entry.getValue().tierName)) { + entry.getKey().put(key, value); + } + } + updateStatsOnPut(cacheValueTuple.v2(), key, value); + } + } + } + + @Override + public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> loader) throws Exception { + // Don't capture stats in the initial getValueFromTieredCache(). If we have concurrent requests for the same key, + // and it only has to be loaded one time, we should report one miss and the rest hits. But, if we do stats in + // getValueFromTieredCache(), + // we will see all misses. Instead, handle stats in computeIfAbsent(). + Tuple cacheValueTuple; + CompletableFuture, V>> future = null; + try (ReleasableLock ignore = readLock.acquire()) { + cacheValueTuple = getValueFromTieredCache(false).apply(key); + if (cacheValueTuple == null) { + // Only one of the threads will succeed putting a future into map for the same key. + // Rest will fetch existing future and wait on that to complete. + future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>()); + } + } + List heapDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_ON_HEAP); + List diskDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_DISK); + + if (cacheValueTuple == null) { + // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside. + // This is needed as there can be many requests for the same key at the same time and we only want to load + // the value once. + V value = compute(key, loader, future); + // Handle stats + if (loader.isLoaded()) { + // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk + // cache + // if present + updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value); + statsHolder.incrementMisses(heapDimensionValues); + if (caches.get(diskCache).isEnabled()) { + statsHolder.incrementMisses(diskDimensionValues); + } + } else { + // Another thread requesting this key already loaded the value. Register a hit for the heap cache + statsHolder.incrementHits(heapDimensionValues); } + return value; } else { - // Another thread requesting this key already loaded the value. Register a hit for the heap cache - statsHolder.incrementHits(heapDimensionValues); + // Handle stats for an initial hit from getValueFromTieredCache() + if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_ON_HEAP)) { + // A hit for the heap tier + statsHolder.incrementHits(heapDimensionValues); + } else if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_DISK)) { + // Miss for the heap tier, hit for the disk tier + statsHolder.incrementMisses(heapDimensionValues); + statsHolder.incrementHits(diskDimensionValues); + } + } + return cacheValueTuple.v1(); + } + + private V compute(ICacheKey key, LoadAwareCacheLoader, V> loader, CompletableFuture, V>> future) + throws Exception { + // Handler to handle results post-processing. Takes a tuple or exception as an input and returns + // the value. Also before returning value, puts the value in cache. + BiFunction, V>, Throwable, Void> handler = (pair, ex) -> { + if (pair != null) { + try (ReleasableLock ignore = writeLock.acquire()) { + onHeapCache.put(pair.v1(), pair.v2()); + } catch (Exception e) { + // TODO: Catch specific exceptions to know whether this resulted from cache or underlying removal + // listeners/stats. Needs better exception handling at underlying layers.For now swallowing + // exception. + logger.warn("Exception occurred while putting item onto heap cache", e); + } + } else { + if (ex != null) { + logger.warn("Exception occurred while trying to compute the value", ex); + } + } + completableFutureMap.remove(key);// Remove key from map as not needed anymore. + return null; + }; + V value = null; + if (future == null) { + future = completableFutureMap.get(key); + future.handle(handler); + try { + value = loader.load(key); + } catch (Exception ex) { + future.completeExceptionally(ex); + throw new ExecutionException(ex); + } + if (value == null) { + NullPointerException npe = new NullPointerException("Loader returned a null value"); + future.completeExceptionally(npe); + throw new ExecutionException(npe); + } else { + future.complete(new Tuple<>(key, value)); + } + } else { + try { + value = future.get().v2(); + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); + } } return value; - } else { - // Handle stats for an initial hit from getValueFromTieredCache() - if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_ON_HEAP)) { - // A hit for the heap tier - statsHolder.incrementHits(heapDimensionValues); - } else if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_DISK)) { - // Miss for the heap tier, hit for the disk tier - statsHolder.incrementMisses(heapDimensionValues); - statsHolder.incrementHits(diskDimensionValues); + } + + @Override + public void invalidate(ICacheKey key) { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + if (key.getDropStatsForDimensions()) { + List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, cacheEntry.getValue().tierName); + statsHolder.removeDimensions(dimensionValues); + } + if (key.key != null) { + try (ReleasableLock ignore = writeLock.acquire()) { + cacheEntry.getKey().invalidate(key); + } + } } } - return cacheValueTuple.v1(); - } - private V compute(ICacheKey key, LoadAwareCacheLoader, V> loader, CompletableFuture, V>> future) - throws Exception { - // Handler to handle results post processing. Takes a tuple or exception as an input and returns - // the value. Also before returning value, puts the value in cache. - BiFunction, V>, Throwable, Void> handler = (pair, ex) -> { - if (pair != null) { - try (ReleasableLock ignore = writeLock.acquire()) { - onHeapCache.put(pair.v1(), pair.v2()); - } catch (Exception e) { - // TODO: Catch specific exceptions to know whether this resulted from cache or underlying removal - // listeners/stats. Needs better exception handling at underlying layers.For now swallowing - // exception. - logger.warn("Exception occurred while putting item onto heap cache", e); + @Override + public void invalidateAll() { + try (ReleasableLock ignore = writeLock.acquire()) { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + cacheEntry.getKey().invalidateAll(); } - } else { - if (ex != null) { - logger.warn("Exception occurred while trying to compute the value", ex); + } + statsHolder.reset(); + } + + @SuppressWarnings({ "unchecked" }) + @Override + public Iterable> keys() { + List>> iterableList = new ArrayList<>(); + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + iterableList.add(cacheEntry.getKey().keys()); + } + Iterable>[] iterables = (Iterable>[]) iterableList.toArray(new Iterable[0]); + return new ConcatenatedIterables<>(iterables); + } + + @Override + public long count() { + return onHeapCache.count() + diskCache.count(); + } + + @Override + public void refresh() { + try (ReleasableLock ignore = writeLock.acquire()) { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + cacheEntry.getKey().refresh(); } } - completableFutureMap.remove(key);// Remove key from map as not needed anymore. + } + + @Override + public ImmutableCacheStatsHolder stats(String[] levels) { return null; - }; - V value = null; - if (future == null) { - future = completableFutureMap.get(key); - future.handle(handler); - try { - value = loader.load(key); - } catch (Exception ex) { - future.completeExceptionally(ex); - throw new ExecutionException(ex); + } + + @Override + public void close() throws IOException { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + // Close all the caches here irrespective of whether they are enabled or not. + cacheEntry.getKey().close(); } - if (value == null) { - NullPointerException npe = new NullPointerException("Loader returned a null value"); - future.completeExceptionally(npe); - throw new ExecutionException(npe); - } else { - future.complete(new Tuple<>(key, value)); + } + + void handleRemovalFromHeapTier(RemovalNotification, V> notification) { + ICacheKey key = notification.getKey(); + boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason()); + boolean countEvictionTowardsTotal = false; // Don't count this eviction towards the cache's total if it ends up in the disk tier + boolean exceptionOccurredOnDiskCachePut = false; + boolean canCacheOnDisk = caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue()); + if (canCacheOnDisk) { + try (ReleasableLock ignore = writeLock.acquire()) { + diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats + } catch (Exception ex) { + // TODO: Catch specific exceptions. Needs better exception handling. We are just swallowing exception + // in this case as it shouldn't cause upstream request to fail. + logger.warn("Exception occurred while putting item to disk cache", ex); + exceptionOccurredOnDiskCachePut = true; + } + if (!exceptionOccurredOnDiskCachePut) { + updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue()); + } } - } else { - try { - value = future.get().v2(); - } catch (InterruptedException ex) { - throw new IllegalStateException(ex); + if (!canCacheOnDisk || exceptionOccurredOnDiskCachePut) { + // If the value is not going to the disk cache, send this notification to the TSC's removal listener + // as the value is leaving the TSC entirely + removalListener.onRemoval(notification); + countEvictionTowardsTotal = true; } + updateStatsOnRemoval(TIER_DIMENSION_VALUE_ON_HEAP, wasEvicted, key, notification.getValue(), countEvictionTowardsTotal); } - return value; - } - @Override - public void invalidate(ICacheKey key) { - // We are trying to invalidate the key from all caches though it would be present in only of them. - // Doing this as we don't know where it is located. We could do a get from both and check that, but what will - // also trigger a hit/miss listener event, so ignoring it for now. - // We don't update stats here, as this is handled by the removal listeners for the tiers. - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - if (key.getDropStatsForDimensions()) { - List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, cacheEntry.getValue().tierName); - statsHolder.removeDimensions(dimensionValues); + boolean evaluatePolicies(V value) { + for (Predicate policy : policies) { + if (!policy.test(value)) { + return false; + } } - if (key.key != null) { - try (ReleasableLock ignore = writeLock.acquire()) { - cacheEntry.getKey().invalidate(key); + return true; + } + + /** + * Get a value from the tiered cache, and the name of the tier it was found in. + * @param captureStats Whether to record hits/misses for this call of the function + * @return A tuple of the value and the name of the tier it was found in. + */ + private Function, Tuple> getValueFromTieredCache(boolean captureStats) { + return key -> { + try (ReleasableLock ignore = readLock.acquire()) { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { + if (cacheEntry.getValue().isEnabled()) { + V value = cacheEntry.getKey().get(key); + // Get the tier value corresponding to this cache + String tierValue = cacheEntry.getValue().tierName; + List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, tierValue); + if (value != null) { + if (captureStats) { + statsHolder.incrementHits(dimensionValues); + } + return new Tuple<>(value, tierValue); + } else if (captureStats) { + statsHolder.incrementMisses(dimensionValues); + } + } + } + return null; } + }; + } + + void handleRemovalFromDiskTier(RemovalNotification, V> notification) { + // Values removed from the disk tier leave the TSC entirely + removalListener.onRemoval(notification); + boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason()); + updateStatsOnRemoval(TIER_DIMENSION_VALUE_DISK, wasEvicted, notification.getKey(), notification.getValue(), true); + } + + void updateStatsOnRemoval( + String removedFromTierValue, + boolean wasEvicted, + ICacheKey key, + V value, + boolean countEvictionTowardsTotal + ) { + List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, removedFromTierValue); + if (wasEvicted) { + statsHolder.incrementEvictions(dimensionValues, countEvictionTowardsTotal); + } + statsHolder.decrementItems(dimensionValues); + statsHolder.decrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value)); + } + + void updateStatsOnPut(String destinationTierValue, ICacheKey key, V value) { + List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, destinationTierValue); + statsHolder.incrementItems(dimensionValues); + statsHolder.incrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value)); + } + + /** + * A class which receives removal events from the heap tier. + */ + private class HeapTierRemovalListener implements RemovalListener, V> { + private final TieredSpilloverCacheSegment tsc; + + HeapTierRemovalListener(TieredSpilloverCacheSegment tsc) { + this.tsc = tsc; + } + + @Override + public void onRemoval(RemovalNotification, V> notification) { + tsc.handleRemovalFromHeapTier(notification); + } + } + + /** + * A class which receives removal events from the disk tier. + */ + private class DiskTierRemovalListener implements RemovalListener, V> { + private final TieredSpilloverCacheSegment tsc; + + DiskTierRemovalListener(TieredSpilloverCacheSegment tsc) { + this.tsc = tsc; + } + + @Override + public void onRemoval(RemovalNotification, V> notification) { + tsc.handleRemovalFromDiskTier(notification); } } } + // Package private for testing. + void enableDisableDiskCache(Boolean isDiskCacheEnabled) { + for (int iter = 0; iter < this.numberOfSegments; iter++) { + tieredSpilloverCacheSegments[iter].enableDisableDiskCache(isDiskCacheEnabled); + } + this.statsHolder.setDiskCacheEnabled(isDiskCacheEnabled); + } + + // Package private for testing. + TieredSpilloverCacheSegment getTieredCacheSegment(ICacheKey key) { + return tieredSpilloverCacheSegments[getSegmentNumber(key)]; + } + + int getSegmentNumber(ICacheKey key) { + return key.hashCode() & (this.numberOfSegments - 1); + } + + int getNumberOfSegments() { + return tieredSpilloverCacheSegments.length; + } + + @Override + public V get(ICacheKey key) { + TieredSpilloverCacheSegment tieredSpilloverCacheSegment = getTieredCacheSegment(key); + return tieredSpilloverCacheSegment.get(key); + } + + @Override + public void put(ICacheKey key, V value) { + TieredSpilloverCacheSegment tieredSpilloverCacheSegment = getTieredCacheSegment(key); + tieredSpilloverCacheSegment.put(key, value); + } + + @Override + public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> loader) throws Exception { + TieredSpilloverCacheSegment tieredSpilloverCacheSegment = getTieredCacheSegment(key); + return tieredSpilloverCacheSegment.computeIfAbsent(key, loader); + } + + @Override + public void invalidate(ICacheKey key) { + TieredSpilloverCacheSegment tieredSpilloverCacheSegment = getTieredCacheSegment(key); + tieredSpilloverCacheSegment.invalidate(key); + } + @Override public void invalidateAll() { - try (ReleasableLock ignore = writeLock.acquire()) { - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - cacheEntry.getKey().invalidateAll(); - } + for (int iter = 0; iter < this.numberOfSegments; iter++) { + tieredSpilloverCacheSegments[iter].invalidateAll(); } - statsHolder.reset(); } /** @@ -327,8 +621,8 @@ public void invalidateAll() { @Override public Iterable> keys() { List>> iterableList = new ArrayList<>(); - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - iterableList.add(cacheEntry.getKey().keys()); + for (int iter = 0; iter < this.numberOfSegments; iter++) { + iterableList.add(tieredSpilloverCacheSegments[iter].keys()); } Iterable>[] iterables = (Iterable>[]) iterableList.toArray(new Iterable[0]); return new ConcatenatedIterables<>(iterables); @@ -343,18 +637,15 @@ public long count() { @Override public void refresh() { - try (ReleasableLock ignore = writeLock.acquire()) { - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - cacheEntry.getKey().refresh(); - } + for (int iter = 0; iter < this.numberOfSegments; iter++) { + tieredSpilloverCacheSegments[iter].refresh(); } } @Override public void close() throws IOException { - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - // Close all the caches here irrespective of whether they are enabled or not. - cacheEntry.getKey().close(); + for (int iter = 0; iter < this.numberOfSegments; iter++) { + tieredSpilloverCacheSegments[iter].close(); } } @@ -363,130 +654,44 @@ public ImmutableCacheStatsHolder stats(String[] levels) { return statsHolder.getImmutableCacheStatsHolder(levels); } - /** - * Get a value from the tiered cache, and the name of the tier it was found in. - * @param captureStats Whether to record hits/misses for this call of the function - * @return A tuple of the value and the name of the tier it was found in. - */ - private Function, Tuple> getValueFromTieredCache(boolean captureStats) { - return key -> { - try (ReleasableLock ignore = readLock.acquire()) { - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - if (cacheEntry.getValue().isEnabled()) { - V value = cacheEntry.getKey().get(key); - // Get the tier value corresponding to this cache - String tierValue = cacheEntry.getValue().tierName; - List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, tierValue); - if (value != null) { - if (captureStats) { - statsHolder.incrementHits(dimensionValues); - } - return new Tuple<>(value, tierValue); - } else if (captureStats) { - statsHolder.incrementMisses(dimensionValues); - } - } - } - return null; - } - }; - } - - void handleRemovalFromHeapTier(RemovalNotification, V> notification) { - ICacheKey key = notification.getKey(); - boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason()); - boolean countEvictionTowardsTotal = false; // Don't count this eviction towards the cache's total if it ends up in the disk tier - boolean exceptionOccurredOnDiskCachePut = false; - boolean canCacheOnDisk = caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue()); - if (canCacheOnDisk) { - try (ReleasableLock ignore = writeLock.acquire()) { - diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats - } catch (Exception ex) { - // TODO: Catch specific exceptions. Needs better exception handling. We are just swallowing exception - // in this case as it shouldn't cause upstream request to fail. - logger.warn("Exception occurred while putting item to disk cache", ex); - exceptionOccurredOnDiskCachePut = true; - } - if (!exceptionOccurredOnDiskCachePut) { - updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue()); - } - } - if (!canCacheOnDisk || exceptionOccurredOnDiskCachePut) { - // If the value is not going to the disk cache, send this notification to the TSC's removal listener - // as the value is leaving the TSC entirely - removalListener.onRemoval(notification); - countEvictionTowardsTotal = true; + // Package private for testing. + @SuppressWarnings({ "unchecked" }) + Iterable> getOnHeapCacheKeys() { + List>> iterableList = new ArrayList<>(); + for (int iter = 0; iter < this.numberOfSegments; iter++) { + iterableList.add(tieredSpilloverCacheSegments[iter].onHeapCache.keys()); } - updateStatsOnRemoval(TIER_DIMENSION_VALUE_ON_HEAP, wasEvicted, key, notification.getValue(), countEvictionTowardsTotal); - } - - void handleRemovalFromDiskTier(RemovalNotification, V> notification) { - // Values removed from the disk tier leave the TSC entirely - removalListener.onRemoval(notification); - boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason()); - updateStatsOnRemoval(TIER_DIMENSION_VALUE_DISK, wasEvicted, notification.getKey(), notification.getValue(), true); - } - - void updateStatsOnRemoval( - String removedFromTierValue, - boolean wasEvicted, - ICacheKey key, - V value, - boolean countEvictionTowardsTotal - ) { - List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, removedFromTierValue); - if (wasEvicted) { - statsHolder.incrementEvictions(dimensionValues, countEvictionTowardsTotal); - } - statsHolder.decrementItems(dimensionValues); - statsHolder.decrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value)); - } - - void updateStatsOnPut(String destinationTierValue, ICacheKey key, V value) { - List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, destinationTierValue); - statsHolder.incrementItems(dimensionValues); - statsHolder.incrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value)); + Iterable>[] iterables = (Iterable>[]) iterableList.toArray(new Iterable[0]); + return new ConcatenatedIterables<>(iterables); } - boolean evaluatePolicies(V value) { - for (Predicate policy : policies) { - if (!policy.test(value)) { - return false; - } + // Package private for testing. + @SuppressWarnings({ "unchecked" }) + Iterable> getDiskCacheKeys() { + List>> iterableList = new ArrayList<>(); + for (int iter = 0; iter < this.numberOfSegments; iter++) { + iterableList.add(tieredSpilloverCacheSegments[iter].diskCache.keys()); } - return true; + Iterable>[] iterables = (Iterable>[]) iterableList.toArray(new Iterable[0]); + return new ConcatenatedIterables<>(iterables); } - /** - * A class which receives removal events from the heap tier. - */ - private class HeapTierRemovalListener implements RemovalListener, V> { - private final TieredSpilloverCache tsc; - - HeapTierRemovalListener(TieredSpilloverCache tsc) { - this.tsc = tsc; - } - - @Override - public void onRemoval(RemovalNotification, V> notification) { - tsc.handleRemovalFromHeapTier(notification); + // Package private for testing. + long onHeapCacheCount() { + long onHeapCacheEntries = 0; + for (int iter = 0; iter < this.numberOfSegments; iter++) { + onHeapCacheEntries += tieredSpilloverCacheSegments[iter].onHeapCache.count(); } + return onHeapCacheEntries; } - /** - * A class which receives removal events from the disk tier. - */ - private class DiskTierRemovalListener implements RemovalListener, V> { - private final TieredSpilloverCache tsc; - - DiskTierRemovalListener(TieredSpilloverCache tsc) { - this.tsc = tsc; - } - - @Override - public void onRemoval(RemovalNotification, V> notification) { - tsc.handleRemovalFromDiskTier(notification); + // Package private for testing. + long diskCacheCount() { + long diskCacheEntries = 0; + for (int iter = 0; iter < this.numberOfSegments; iter++) { + diskCacheEntries += tieredSpilloverCacheSegments[iter].diskCache.count(); } + return diskCacheEntries; } /** @@ -550,7 +755,7 @@ public void remove() { } } - private class TierInfo { + private static class TierInfo { AtomicBoolean isEnabled; final String tierName; @@ -611,12 +816,29 @@ public ICache create(CacheConfig config, CacheType cacheType, "Cached result parser fn can't be null" ); + int numberOfSegments = TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(cacheType.getSettingPrefix()).get(settings); + + if (!VALID_SEGMENT_COUNT_VALUES.contains(numberOfSegments)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE, TIERED_SPILLOVER_CACHE_NAME) + ); + } + + long onHeapCacheSize = TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) + .get(settings) + .getBytes(); + long diskCacheSize = TIERED_SPILLOVER_DISK_STORE_SIZE.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) + .get(settings); + return new Builder().setDiskCacheFactory(diskCacheFactory) .setOnHeapCacheFactory(onHeapCacheFactory) .setRemovalListener(config.getRemovalListener()) .setCacheConfig(config) .setCacheType(cacheType) + .setNumberOfSegments(numberOfSegments) .addPolicy(new TookTimePolicy(diskPolicyThreshold, cachedResultParser, config.getClusterSettings(), cacheType)) + .setOnHeapCacheSizeInBytes(onHeapCacheSize) + .setDiskCacheSize(diskCacheSize) .build(); } @@ -640,6 +862,10 @@ public static class Builder { private Map cacheFactories; private final ArrayList> policies = new ArrayList<>(); + private int numberOfSegments; + private long onHeapCacheSizeInBytes; + private long diskCacheSizeInBytes; + /** * Default constructor */ @@ -725,6 +951,36 @@ public Builder addPolicies(List> policies) { return this; } + /** + * Sets number of segments for tiered cache + * @param numberOfSegments number of segments + * @return builder + */ + public Builder setNumberOfSegments(int numberOfSegments) { + this.numberOfSegments = numberOfSegments; + return this; + } + + /** + * Sets onHeap cache size + * @param onHeapCacheSizeInBytes size of onHeap cache in bytes + * @return builder + */ + public Builder setOnHeapCacheSizeInBytes(long onHeapCacheSizeInBytes) { + this.onHeapCacheSizeInBytes = onHeapCacheSizeInBytes; + return this; + } + + /** + * Sets disk cache siz + * @param diskCacheSizeInBytes size of diskCache in bytes + * @return buider + */ + public Builder setDiskCacheSize(long diskCacheSizeInBytes) { + this.diskCacheSizeInBytes = diskCacheSizeInBytes; + return this; + } + /** * Build tiered spillover cache. * @return TieredSpilloverCache diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java index 1c10e51630460..bf522b42b70ca 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java @@ -65,6 +65,15 @@ public List> getSettings() { if (FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings)) { settingList.add(DISK_CACHE_ENABLED_SETTING_MAP.get(cacheType)); } + settingList.add( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) + ); + settingList.add( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) + ); + settingList.add( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_SIZE.getConcreteSettingForNamespace(cacheType.getSettingPrefix()) + ); } return settingList; } diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java index e8e441d6bd3a6..122d00af3bd1e 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java @@ -11,11 +11,16 @@ import org.opensearch.common.cache.CacheType; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.threadpool.ThreadPool; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.opensearch.common.cache.settings.CacheSettings.INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE; +import static org.opensearch.common.cache.settings.CacheSettings.VALID_SEGMENT_COUNT_VALUES; import static org.opensearch.common.settings.Setting.Property.NodeScope; /** @@ -23,6 +28,16 @@ */ public class TieredSpilloverCacheSettings { + /** + * Default cache size in bytes ie 1gb. + */ + public static final long DEFAULT_DISK_CACHE_SIZE_IN_BYTES = 1073741824L; + + /** + * Minimum disk cache size ie 10mb. May not make such sense to keep a value smaller than this. + */ + public static final long MIN_DISK_CACHE_SIZE_IN_BYTES = 10485760L; + /** * Setting which defines the onHeap cache store to be used in TieredSpilloverCache. * @@ -50,6 +65,43 @@ public class TieredSpilloverCacheSettings { (key) -> Setting.boolSetting(key, true, NodeScope, Setting.Property.Dynamic) ); + /** + * Setting defining the number of segments within Tiered cache + */ + public static final Setting.AffixSetting TIERED_SPILLOVER_SEGMENTS = Setting.suffixKeySetting( + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ".segments", + (key) -> Setting.intSetting(key, defaultSegments(), 1, k -> { + if (!VALID_SEGMENT_COUNT_VALUES.contains(k)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE, + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + ); + } + }, NodeScope) + ); + + /** + * Setting which defines the onHeap cache size to be used within tiered cache. + * + * Pattern: {cache_type}.tiered_spillover.onheap.store.size + * Example: indices.request.cache.tiered_spillover.onheap.store.size + */ + public static final Setting.AffixSetting TIERED_SPILLOVER_ONHEAP_STORE_SIZE = Setting.suffixKeySetting( + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ".onheap.store.size", + (key) -> Setting.memorySizeSetting(key, "1%", NodeScope) + ); + + /** + * Setting which defines the disk cache size to be used within tiered cache. + */ + public static final Setting.AffixSetting TIERED_SPILLOVER_DISK_STORE_SIZE = Setting.suffixKeySetting( + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ".disk.store.size", + (key) -> Setting.longSetting(key, DEFAULT_DISK_CACHE_SIZE_IN_BYTES, MIN_DISK_CACHE_SIZE_IN_BYTES, NodeScope) + ); + /** * Setting defining the minimum took time for a query to be allowed into the disk cache. */ @@ -96,6 +148,23 @@ public class TieredSpilloverCacheSettings { DISK_CACHE_ENABLED_SETTING_MAP = diskCacheSettingMap; } + /** + * Returns the default segment count to be used within TieredCache. + * @return default segment count + */ + public static int defaultSegments() { + // For now, we use number of search threads as the default segment count. If needed each cache type can + // configure its own segmentCount via setting in the future. + int defaultSegmentCount = ThreadPool.searchThreadPoolSize(Runtime.getRuntime().availableProcessors()); + // Now round it off to the next power of 2 as we don't support any other values. + for (int segmentValue : VALID_SEGMENT_COUNT_VALUES) { + if (defaultSegmentCount <= segmentValue) { + return segmentValue; + } + } + return VALID_SEGMENT_COUNT_VALUES.get(VALID_SEGMENT_COUNT_VALUES.size() - 1); + } + /** * Default constructor */ diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java index 69e2060f7ea2f..fcddd489a27aa 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java @@ -62,6 +62,7 @@ public void put(ICacheKey key, V value) { if (this.cache.size() >= maxSize) { // For simplification this.removalListener.onRemoval(new RemovalNotification<>(key, value, RemovalReason.EVICTED)); this.statsHolder.decrementItems(List.of()); + return; } try { Thread.sleep(delay); @@ -86,8 +87,10 @@ public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> @Override public void invalidate(ICacheKey key) { - removalListener.onRemoval(new RemovalNotification<>(key, cache.get(key), RemovalReason.INVALIDATED)); - this.cache.remove(key); + V value = this.cache.remove(key); + if (value != null) { + removalListener.onRemoval(new RemovalNotification<>(key, cache.get(key), RemovalReason.INVALIDATED)); + } } @Override @@ -131,11 +134,13 @@ public static class MockDiskCacheFactory implements Factory { final long delay; final int maxSize; final boolean statsTrackingEnabled; + final int keyValueSize; - public MockDiskCacheFactory(long delay, int maxSize, boolean statsTrackingEnabled) { + public MockDiskCacheFactory(long delay, int maxSize, boolean statsTrackingEnabled, int keyValueSize) { this.delay = delay; this.maxSize = maxSize; this.statsTrackingEnabled = statsTrackingEnabled; + this.keyValueSize = keyValueSize; } @Override @@ -145,13 +150,21 @@ public ICache create(CacheConfig config, CacheType cacheType, // cache would require. assert config.getKeySerializer() != null; assert config.getValueSerializer() != null; - return new Builder().setKeySerializer((Serializer) config.getKeySerializer()) + MockDiskCache.Builder builder = (Builder) new Builder().setKeySerializer( + (Serializer) config.getKeySerializer() + ) .setValueSerializer((Serializer) config.getValueSerializer()) - .setMaxSize(maxSize) .setDeliberateDelay(delay) .setRemovalListener(config.getRemovalListener()) - .setStatsTrackingEnabled(config.getStatsTrackingEnabled()) - .build(); + .setStatsTrackingEnabled(config.getStatsTrackingEnabled()); + + // For mock disk cache, size refers to number of entries for simplicity. + if (config.getMaxSizeInBytes() > 0) { + builder.setMaxSize(Math.toIntExact(config.getMaxSizeInBytes())); + } else { + builder.setMaxSize(maxSize); + } + return builder.build(); } @Override diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java index c6440a1e1797f..1215a2130ac2d 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java @@ -8,6 +8,7 @@ package org.opensearch.cache.common.tier; +import org.opensearch.OpenSearchException; import org.opensearch.common.Randomness; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.ICache; @@ -29,6 +30,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.env.NodeEnvironment; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -54,11 +56,16 @@ import java.util.function.Function; import java.util.function.Predicate; +import static org.opensearch.cache.common.tier.TieredSpilloverCache.ZERO_SEGMENT_COUNT_EXCEPTION_MESSAGE; import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS; import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP; +import static org.opensearch.common.cache.settings.CacheSettings.INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE; +import static org.opensearch.common.cache.settings.CacheSettings.VALID_SEGMENT_COUNT_VALUES; import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -89,13 +96,14 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + 1 ); int numOfItems1 = randomIntBetween(1, onHeapCacheSize / 2 - 1); List> keys = new ArrayList<>(); @@ -160,18 +168,19 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception MockDiskCache.MockDiskCacheFactory.NAME ) .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .put( CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) + .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 1) .put(FeatureFlags.PLUGGABLE_CACHE, "true") .build(); - + String storagePath = getStoragePath(settings); ICache tieredSpilloverICache = new TieredSpilloverCache.TieredSpilloverCacheFactory().create( new CacheConfig.Builder().setKeyType(String.class) .setKeyType(String.class) @@ -182,15 +191,16 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception .setSettings(settings) .setDimensionNames(dimensionNames) .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken - // 20_000_000 ns = 20 ms to compute + // 20_000_000 ns = 20 ms to compute .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) .build(), CacheType.INDICES_REQUEST_CACHE, Map.of( OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), MockDiskCache.MockDiskCacheFactory.NAME, - new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false) + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) ) ); @@ -206,10 +216,10 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception } int expectedDiskEntries = numOfItems1 - onHeapCacheSize; - tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add); - tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getOnHeapCache().keys().forEach(onHeapKeys::add); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getDiskCache().keys().forEach(diskTierKeys::add); // Verify on heap cache stats. - assertEquals(onHeapCacheSize, tieredSpilloverCache.getOnHeapCache().count()); + assertEquals(onHeapCacheSize, tieredSpilloverCache.tieredSpilloverCacheSegments[0].getOnHeapCache().count()); assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); @@ -217,7 +227,7 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); // Verify disk cache stats. - assertEquals(expectedDiskEntries, tieredSpilloverCache.getDiskCache().count()); + assertEquals(expectedDiskEntries, tieredSpilloverCache.tieredSpilloverCacheSegments[0].getDiskCache().count()); assertEquals(expectedDiskEntries, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); @@ -225,6 +235,149 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception assertEquals(expectedDiskEntries * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); } + public void testComputeIfAbsentWithSegmentedCache() throws Exception { + int onHeapCacheSize = randomIntBetween(300, 600); + int diskCacheSize = randomIntBetween(700, 1200); + int totalSize = onHeapCacheSize + diskCacheSize; + int numberOfSegments = getNumberOfSegments(); + int keyValueSize = 11; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + int onHeapCacheSizeInBytes = onHeapCacheSize * keyValueSize; + Map expectedSegmentOnHeapCacheSize = getSegmentOnHeapCacheSize( + numberOfSegments, + onHeapCacheSizeInBytes, + keyValueSize + ); + int totalOnHeapEntries = 0; + int totalOnDiskEntries = 0; + // Set the desired settings needed to create a TieredSpilloverCache object with INDICES_REQUEST_CACHE cacheType. + Settings settings = Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME + ) + .put( + TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), + numberOfSegments + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + MockDiskCache.MockDiskCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSizeInBytes + "b" + ) + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .build(); + String storagePath = getStoragePath(settings); + ICache tieredSpilloverICache = new TieredSpilloverCache.TieredSpilloverCacheFactory().create( + new CacheConfig.Builder().setKeyType(String.class) + .setKeyType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setRemovalListener(removalListener) + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setSettings(settings) + .setDimensionNames(dimensionNames) + .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken + // 20_000_000 ns = 20 ms to compute + .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) + .setSegmentCount(numberOfSegments) + .build(), + CacheType.INDICES_REQUEST_CACHE, + Map.of( + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, + new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), + MockDiskCache.MockDiskCacheFactory.NAME, + // Make disk cache big enough to hold all entries + new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize * 500, false, keyValueSize) + ) + ); + TieredSpilloverCache tieredSpilloverCache = (TieredSpilloverCache) tieredSpilloverICache; + TieredSpilloverCache.TieredSpilloverCacheSegment[] tieredSpilloverCacheSegments = + tieredSpilloverCache.tieredSpilloverCacheSegments; + assertEquals(numberOfSegments, tieredSpilloverCacheSegments.length); + + Map, TieredSpilloverCache.TieredSpilloverCacheSegment> tieredSpilloverCacheSegmentMap = + new HashMap<>(); + + int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize); + List> onHeapKeys = new ArrayList<>(); + List> diskTierKeys = new ArrayList<>(); + Map expectedNumberOfEntriesInSegment = new HashMap<>(); + for (int iter = 0; iter < numOfItems1; iter++) { + String key = UUID.randomUUID().toString(); + LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); + ICacheKey iCacheKey = getICacheKey(key); + int keySegment = tieredSpilloverCache.getSegmentNumber(iCacheKey); + if (expectedNumberOfEntriesInSegment.get(keySegment) == null) { + expectedNumberOfEntriesInSegment.put(keySegment, Integer.valueOf(1)); + } else { + Integer updatedValue = expectedNumberOfEntriesInSegment.get(keySegment) + 1; + expectedNumberOfEntriesInSegment.put(keySegment, updatedValue); + } + tieredSpilloverCacheSegmentMap.put(iCacheKey, tieredSpilloverCache.getTieredCacheSegment(iCacheKey)); + tieredSpilloverCache.computeIfAbsent(iCacheKey, tieredCacheLoader); + } + + // We now calculate expected onHeap cache entries and then verify it later. + for (int i = 0; i < numberOfSegments; i++) { + if (expectedNumberOfEntriesInSegment.get(i) == null) { + continue; + } + if (expectedNumberOfEntriesInSegment.get(i) >= expectedSegmentOnHeapCacheSize.get(i)) { + totalOnHeapEntries += expectedSegmentOnHeapCacheSize.get(i); + totalOnDiskEntries += expectedNumberOfEntriesInSegment.get(i) - expectedSegmentOnHeapCacheSize.get(i); + } else { + // In this case onHeap cache wasn't utilized fully. + totalOnHeapEntries += expectedNumberOfEntriesInSegment.get(i); + } + } + + tieredSpilloverCache.getOnHeapCacheKeys().forEach(onHeapKeys::add); + tieredSpilloverCache.getDiskCacheKeys().forEach(diskTierKeys::add); + // Verify on heap cache stats. + assertEquals(totalOnHeapEntries, tieredSpilloverCache.onHeapCacheCount()); + assertEquals(totalOnHeapEntries, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(totalOnDiskEntries, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(totalOnHeapEntries * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + + // Verify disk cache stats. + assertEquals(totalOnDiskEntries, tieredSpilloverCache.diskCacheCount()); + assertEquals(totalOnDiskEntries, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(totalOnDiskEntries * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + + // Verify the keys for onHeap and disk cache + + for (ICacheKey key : onHeapKeys) { + assertNotNull(tieredSpilloverCache.get(key)); + } + assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(onHeapKeys.size(), getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + for (ICacheKey key : diskTierKeys) { + assertNotNull(tieredSpilloverCache.get(key)); + } + assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(diskTierKeys.size(), getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + } + public void testWithFactoryCreationWithOnHeapCacheNotPresent() { int onHeapCacheSize = randomIntBetween(10, 30); int keyValueSize = 50; @@ -239,9 +392,9 @@ public void testWithFactoryCreationWithOnHeapCacheNotPresent() { MockDiskCache.MockDiskCacheFactory.NAME ) .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .put( @@ -265,7 +418,7 @@ public void testWithFactoryCreationWithOnHeapCacheNotPresent() { OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), MockDiskCache.MockDiskCacheFactory.NAME, - new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false) + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) ) ) ); @@ -289,9 +442,9 @@ public void testWithFactoryCreationWithDiskCacheNotPresent() { OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME ) .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(); @@ -310,7 +463,7 @@ public void testWithFactoryCreationWithDiskCacheNotPresent() { OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), MockDiskCache.MockDiskCacheFactory.NAME, - new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false) + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) ) ) ); @@ -327,6 +480,20 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); + + Settings settings = Settings.builder() + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(); CacheConfig cacheConfig = new CacheConfig.Builder().setKeyType(String.class) .setKeyType(String.class) .setWeigher((k, v) -> keyValueSize) @@ -334,25 +501,12 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { .setKeySerializer(new StringSerializer()) .setValueSerializer(new StringSerializer()) .setDimensionNames(dimensionNames) - .setSettings( - Settings.builder() - .put( - CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), - TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME - ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") - .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), - onHeapCacheSize * keyValueSize + "b" - ) - .build() - ) + .setSettings(settings) + .setStoragePath(getStoragePath(settings)) .setClusterSettings(clusterSettings) .build(); - ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false); + ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false, keyValueSize); TieredSpilloverCache tieredSpilloverCache = new TieredSpilloverCache.Builder() .setOnHeapCacheFactory(onHeapCacheFactory) @@ -360,6 +514,9 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { .setCacheConfig(cacheConfig) .setRemovalListener(removalListener) .setCacheType(CacheType.INDICES_REQUEST_CACHE) + .setNumberOfSegments(1) + .setOnHeapCacheSizeInBytes(onHeapCacheSize * keyValueSize) + .setDiskCacheSize(diskCacheSize) .build(); // Put values in cache more than it's size and cause evictions from onHeap. @@ -372,7 +529,7 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader); } - long actualDiskCacheSize = tieredSpilloverCache.getDiskCache().count(); + long actualDiskCacheSize = tieredSpilloverCache.diskCacheCount(); assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); @@ -381,8 +538,8 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(actualDiskCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); - tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add); - tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add); + tieredSpilloverCache.getOnHeapCacheKeys().forEach(onHeapKeys::add); + tieredSpilloverCache.getDiskCacheKeys().forEach(diskTierKeys::add); // Try to hit cache again with some randomization. int numOfItems2 = randomIntBetween(50, 200); @@ -420,11 +577,11 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { } public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception { - int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100); + int onHeapCacheSize = randomIntBetween(300, 600); + int diskCacheSize = randomIntBetween(700, 1200); int totalSize = onHeapCacheSize + diskCacheSize; - int keyValueSize = 50; - + int numberOfSegments = getNumberOfSegments(); + int keyValueSize = 11; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( keyValueSize, @@ -432,25 +589,53 @@ public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + numberOfSegments + ); + Map onHeapCacheExpectedSize = getSegmentOnHeapCacheSize( + numberOfSegments, + onHeapCacheSize * keyValueSize, + keyValueSize ); + Map mockDiskCacheExpectedSize = getSegmentMockDiskCacheSize(numberOfSegments, diskCacheSize); + Map perSegmentEntryCapacity = new HashMap<>(); + for (int i = 0; i < numberOfSegments; i++) { + int totalEntriesForSegment = onHeapCacheExpectedSize.get(i) + mockDiskCacheExpectedSize.get(i); + perSegmentEntryCapacity.put(i, totalEntriesForSegment); + } int numOfItems = randomIntBetween(totalSize + 1, totalSize * 3); + Map segmentSizeTracker = new HashMap<>(); + int expectedEvictions = 0; for (int iter = 0; iter < numOfItems; iter++) { + ICacheKey iCacheKey = getICacheKey(UUID.randomUUID().toString()); + int keySegment = tieredSpilloverCache.getSegmentNumber(iCacheKey); LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); - tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader); + tieredSpilloverCache.computeIfAbsent(iCacheKey, tieredCacheLoader); + if (segmentSizeTracker.get(keySegment) == null) { + segmentSizeTracker.put(keySegment, Integer.valueOf(1)); + } else { + Integer updatedValue = segmentSizeTracker.get(keySegment) + 1; + segmentSizeTracker.put(keySegment, updatedValue); + } } - - int evictions = numOfItems - (totalSize); // Evictions from the cache as a whole - assertEquals(evictions, removalListener.evictionsMetric.count()); - assertEquals(evictions, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + for (int i = 0; i < numberOfSegments; i++) { + if (segmentSizeTracker.get(i) == null) { + continue; + } + if (segmentSizeTracker.get(i) > perSegmentEntryCapacity.get(i)) { + expectedEvictions += segmentSizeTracker.get(i) - perSegmentEntryCapacity.get(i); + } + } + assertEquals(expectedEvictions, removalListener.evictionsMetric.count()); + assertEquals(expectedEvictions, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); assertEquals( - evictions + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK), + expectedEvictions + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK), getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP) ); } @@ -468,13 +653,14 @@ public void testGetAndCount() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + 1 ); int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize); @@ -509,8 +695,9 @@ public void testGetAndCount() throws Exception { } public void testPut() throws Exception { - int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100); + int numberOfSegments = getNumberOfSegments(); + int onHeapCacheSize = randomIntBetween(10, 30) * numberOfSegments; + int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, onHeapCacheSize * 2); int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); @@ -520,13 +707,14 @@ public void testPut() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + numberOfSegments ); ICacheKey key = getICacheKey(UUID.randomUUID().toString()); String value = UUID.randomUUID().toString(); @@ -541,7 +729,7 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception { int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); - + int numberOfSegments = 1; TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( keyValueSize, diskCacheSize, @@ -552,13 +740,14 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception { TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME ) .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), (onHeapCacheSize * keyValueSize) + "b" ) .build(), - 0 + 0, + numberOfSegments ); for (int i = 0; i < onHeapCacheSize; i++) { @@ -580,7 +769,7 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception { // Verify that new items are part of onHeap cache. List> actualOnHeapCacheKeys = new ArrayList<>(); - tieredSpilloverCache.getOnHeapCache().keys().forEach(actualOnHeapCacheKeys::add); + tieredSpilloverCache.getOnHeapCacheKeys().forEach(actualOnHeapCacheKeys::add); assertEquals(newKeyList.size(), actualOnHeapCacheKeys.size()); for (int i = 0; i < actualOnHeapCacheKeys.size(); i++) { @@ -594,7 +783,6 @@ public void testInvalidate() throws Exception { int onHeapCacheSize = 1; int diskCacheSize = 10; int keyValueSize = 20; - MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( keyValueSize, @@ -602,13 +790,14 @@ public void testInvalidate() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + 1 ); ICacheKey key = getICacheKey(UUID.randomUUID().toString()); String value = UUID.randomUUID().toString(); @@ -652,13 +841,14 @@ public void testCacheKeys() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + 1 ); List> onHeapKeys = new ArrayList<>(); List> diskTierKeys = new ArrayList<>(); @@ -678,8 +868,8 @@ public void testCacheKeys() throws Exception { List> actualOnHeapKeys = new ArrayList<>(); List> actualOnDiskKeys = new ArrayList<>(); - Iterable> onHeapiterable = tieredSpilloverCache.getOnHeapCache().keys(); - Iterable> onDiskiterable = tieredSpilloverCache.getDiskCache().keys(); + Iterable> onHeapiterable = tieredSpilloverCache.getOnHeapCacheKeys(); + Iterable> onDiskiterable = tieredSpilloverCache.getDiskCacheKeys(); onHeapiterable.iterator().forEachRemaining(actualOnHeapKeys::add); onDiskiterable.iterator().forEachRemaining(actualOnDiskKeys::add); for (ICacheKey onHeapKey : onHeapKeys) { @@ -713,16 +903,18 @@ public void testRefresh() { diskCacheSize, removalListener, Settings.EMPTY, - 0 + 0, + 1 ); tieredSpilloverCache.refresh(); } public void testInvalidateAll() throws Exception { - int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(60, 100); - int keyValueSize = 50; + int onHeapCacheSize = randomIntBetween(300, 600); + int diskCacheSize = randomIntBetween(700, 1200); int totalSize = onHeapCacheSize + diskCacheSize; + int numberOfSegments = getNumberOfSegments(); + int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( @@ -731,30 +923,29 @@ public void testInvalidateAll() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + numberOfSegments ); // Put values in cache more than it's size and cause evictions from onHeap. int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize); - List> onHeapKeys = new ArrayList<>(); - List> diskTierKeys = new ArrayList<>(); for (int iter = 0; iter < numOfItems1; iter++) { ICacheKey key = getICacheKey(UUID.randomUUID().toString()); - if (iter > (onHeapCacheSize - 1)) { - // All these are bound to go to disk based cache. - diskTierKeys.add(key); - } else { - onHeapKeys.add(key); - } LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader); } - assertEquals(numOfItems1, tieredSpilloverCache.count()); + assertEquals( + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP) + getItemsForTier( + tieredSpilloverCache, + TIER_DIMENSION_VALUE_DISK + ), + tieredSpilloverCache.count() + ); tieredSpilloverCache.invalidateAll(); assertEquals(0, tieredSpilloverCache.count()); } @@ -767,9 +958,9 @@ public void testComputeIfAbsentConcurrently() throws Exception { MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); Settings settings = Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(); @@ -779,7 +970,8 @@ public void testComputeIfAbsentConcurrently() throws Exception { diskCacheSize, removalListener, settings, - 0 + 0, + 1 ); int numberOfSameKeys = randomIntBetween(400, onHeapCacheSize - 1); @@ -837,16 +1029,17 @@ public String load(ICacheKey key) { } public void testComputIfAbsentConcurrentlyWithMultipleKeys() throws Exception { - int onHeapCacheSize = randomIntBetween(300, 500); + int numberOfSegments = getNumberOfSegments(); + int onHeapCacheSize = randomIntBetween(300, 500) * numberOfSegments; // Able to support all keys in case of + // skewness as well. int diskCacheSize = randomIntBetween(600, 700); int keyValueSize = 50; - MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); Settings settings = Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(); @@ -856,7 +1049,8 @@ public void testComputIfAbsentConcurrentlyWithMultipleKeys() throws Exception { diskCacheSize, removalListener, settings, - 0 + 0, + numberOfSegments ); int iterations = 10; @@ -942,9 +1136,9 @@ public void testComputeIfAbsentWithOnHeapCacheThrowingExceptionOnPut() throws Ex MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); Settings settings = Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(); @@ -952,15 +1146,18 @@ public void testComputeIfAbsentWithOnHeapCacheThrowingExceptionOnPut() throws Ex ICache mockOnHeapCache = mock(ICache.class); when(onHeapCacheFactory.create(any(), any(), any())).thenReturn(mockOnHeapCache); doThrow(new RuntimeException("Testing")).when(mockOnHeapCache).put(any(), any()); - CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener); - ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false); + CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener, 1); + ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false, keyValueSize); TieredSpilloverCache tieredSpilloverCache = getTieredSpilloverCache( onHeapCacheFactory, mockDiskCacheFactory, cacheConfig, null, - removalListener + removalListener, + 1, + onHeapCacheSize * keyValueSize, + diskCacheSize ); String value = ""; value = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() { @@ -980,20 +1177,20 @@ public String load(ICacheKey key) { @SuppressWarnings({ "rawtypes", "unchecked" }) public void testComputeIfAbsentWithDiskCacheThrowingExceptionOnPut() throws Exception { - int onHeapCacheSize = 0; + int onHeapCacheSize = 1; int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); Settings settings = Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(); ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); - CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener); + CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener, 1); ICache.Factory mockDiskCacheFactory = mock(MockDiskCache.MockDiskCacheFactory.class); ICache mockDiskCache = mock(ICache.class); when(mockDiskCacheFactory.create(any(), any(), any())).thenReturn(mockDiskCache); @@ -1004,10 +1201,25 @@ public void testComputeIfAbsentWithDiskCacheThrowingExceptionOnPut() throws Exce mockDiskCacheFactory, cacheConfig, null, - removalListener + removalListener, + 1, + onHeapCacheSize * keyValueSize, + 200 ); String response = ""; + // This first computeIfAbsent ensures onHeap cache has 1 item present and rest will be evicted to disk. + tieredSpilloverCache.computeIfAbsent(getICacheKey("test1"), new LoadAwareCacheLoader<>() { + @Override + public boolean isLoaded() { + return false; + } + + @Override + public String load(ICacheKey key) { + return "test1"; + } + }); response = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() { @Override public boolean isLoaded() { @@ -1050,29 +1262,37 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); - ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize, false); + ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize, false, 1); + + Settings settings = Settings.builder() + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + 200 + "b" + ) + .build(); + String storagePath; + try (NodeEnvironment environment = newNodeEnvironment(settings)) { + storagePath = environment.nodePaths()[0].path + "/test"; + } catch (IOException e) { + throw new OpenSearchException("Exception occurred", e); + } + CacheConfig cacheConfig = new CacheConfig.Builder().setKeyType(String.class) .setKeyType(String.class) .setWeigher((k, v) -> 150) .setRemovalListener(removalListener) .setKeySerializer(new StringSerializer()) .setValueSerializer(new StringSerializer()) - .setSettings( - Settings.builder() - .put( - CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), - TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME - ) - .put(FeatureFlags.PLUGGABLE_CACHE, "true") - .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), - 200 + "b" - ) - .build() - ) + .setSettings(settings) .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) .setDimensionNames(dimensionNames) .build(); TieredSpilloverCache tieredSpilloverCache = new TieredSpilloverCache.Builder() @@ -1080,6 +1300,9 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio .setDiskCacheFactory(diskCacheFactory) .setRemovalListener(removalListener) .setCacheConfig(cacheConfig) + .setOnHeapCacheSizeInBytes(200) + .setDiskCacheSize(diskCacheSize) + .setNumberOfSegments(1) .setCacheType(CacheType.INDICES_REQUEST_CACHE) .build(); @@ -1104,9 +1327,7 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio }); thread.start(); assertBusy(() -> { assertTrue(loadAwareCacheLoader.isLoaded()); }, 100, TimeUnit.MILLISECONDS); // We wait for new key to be loaded - // after which it eviction flow is - // guaranteed to occur. - ICache onDiskCache = tieredSpilloverCache.getDiskCache(); + // after which it eviction flow is guaranteed to occur. // Now on a different thread, try to get key(above one which got evicted) from tiered cache. We expect this // should return not null value as it should be present on diskCache. @@ -1124,13 +1345,13 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio assertNotNull(actualValue.get()); countDownLatch1.await(); - assertEquals(1, tieredSpilloverCache.getOnHeapCache().count()); - assertEquals(1, onDiskCache.count()); + assertEquals(1, tieredSpilloverCache.onHeapCacheCount()); + assertEquals(1, tieredSpilloverCache.diskCacheCount()); assertEquals(1, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); - assertNotNull(onDiskCache.get(keyToBeEvicted)); + assertNotNull(tieredSpilloverCache.getTieredCacheSegment(keyToBeEvicted).getDiskCache().get(keyToBeEvicted)); } public void testDiskTierPolicies() throws Exception { @@ -1140,7 +1361,6 @@ public void testDiskTierPolicies() throws Exception { policies.add(new AllowEvenLengths()); int keyValueSize = 50; - int onHeapCacheSize = 0; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = intializeTieredSpilloverCache( keyValueSize, @@ -1148,14 +1368,15 @@ public void testDiskTierPolicies() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), - onHeapCacheSize * keyValueSize + "b" + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + keyValueSize - 1 + "b" ) .build(), 0, - policies + policies, + 1 ); Map keyValuePairs = new HashMap<>(); @@ -1233,15 +1454,16 @@ public void testTookTimePolicyFromFactory() throws Exception { MockDiskCache.MockDiskCacheFactory.NAME ) .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .put( TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), new TimeValue(timeValueThresholdNanos / 1_000_000) ) + .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 1) .build(); ICache tieredSpilloverICache = new TieredSpilloverCache.TieredSpilloverCacheFactory().create( @@ -1261,13 +1483,14 @@ public CachedQueryResult.PolicyValues apply(String s) { } }) .setClusterSettings(clusterSettings) + .setStoragePath(getStoragePath(settings)) .build(), CacheType.INDICES_REQUEST_CACHE, Map.of( OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), MockDiskCache.MockDiskCacheFactory.NAME, - new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false) + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) ) ); @@ -1279,13 +1502,14 @@ public CachedQueryResult.PolicyValues apply(String s) { } assertEquals(tookTimeMap.size(), tieredSpilloverCache.count()); - // Ensure all these keys get evicted from the on heap tier by adding > heap tier size worth of random keys + // Ensure all these keys get evicted from the on heap tier by adding > heap tier size worth of random keys (this works as we have 1 + // segment) for (int i = 0; i < onHeapCacheSize; i++) { tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), getLoadAwareCacheLoader(keyValueMap)); } - ICache onHeapCache = tieredSpilloverCache.getOnHeapCache(); for (String key : tookTimeMap.keySet()) { - assertNull(onHeapCache.get(getICacheKey(key))); + ICacheKey iCacheKey = getICacheKey(key); + assertNull(tieredSpilloverCache.getTieredCacheSegment(iCacheKey).getOnHeapCache().get(iCacheKey)); } // Now the original keys should be in the disk tier if the policy allows them, or misses if not @@ -1320,10 +1544,10 @@ public void testMinimumThresholdSettingValue() throws Exception { public void testPutWithDiskCacheDisabledSetting() throws Exception { int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100); + int diskCacheSize = randomIntBetween(300, 500); int keyValueSize = 50; int totalSize = onHeapCacheSize + diskCacheSize; - + int numberOfSegments = getNumberOfSegments(); MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( keyValueSize, @@ -1331,14 +1555,15 @@ public void testPutWithDiskCacheDisabledSetting() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false) .build(), - 0 + 0, + numberOfSegments ); int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize); // Create more items than onHeap cache. @@ -1347,59 +1572,95 @@ public void testPutWithDiskCacheDisabledSetting() throws Exception { LoadAwareCacheLoader, String> loadAwareCacheLoader = getLoadAwareCacheLoader(); tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader); } - ICache onHeapCache = tieredSpilloverCache.getOnHeapCache(); - ICache diskCache = tieredSpilloverCache.getDiskCache(); - assertEquals(onHeapCacheSize, onHeapCache.count()); - assertEquals(0, diskCache.count()); // Disk cache shouldn't have anything considering it is disabled. - assertEquals(numOfItems1 - onHeapCacheSize, removalListener.evictionsMetric.count()); + + assertEquals(getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP), tieredSpilloverCache.onHeapCacheCount()); + assertEquals(0, tieredSpilloverCache.diskCacheCount()); // Disk cache shouldn't have anything considering it is + // disabled. + assertEquals(numOfItems1 - tieredSpilloverCache.onHeapCacheCount(), removalListener.evictionsMetric.count()); } public void testGetPutAndInvalidateWithDiskCacheDisabled() throws Exception { - int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100); - int keyValueSize = 50; + int onHeapCacheSize = randomIntBetween(300, 400); + int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 500); + int keyValueSize = 12; int totalSize = onHeapCacheSize + diskCacheSize; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + int numberOfSegments = getNumberOfSegments(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( keyValueSize, diskCacheSize, removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + numberOfSegments ); - + Map onHeapCacheExpectedSize = getSegmentOnHeapCacheSize( + numberOfSegments, + onHeapCacheSize * keyValueSize, + keyValueSize + ); + Map mockDiskCacheExpectedSize = getSegmentMockDiskCacheSize(numberOfSegments, diskCacheSize); + Map perSegmentEntryCapacity = new HashMap<>(); + for (int i = 0; i < numberOfSegments; i++) { + int totalEntriesForSegment = onHeapCacheExpectedSize.get(i) + mockDiskCacheExpectedSize.get(i); + perSegmentEntryCapacity.put(i, totalEntriesForSegment); + } int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize - 1); // Create more items than onHeap // cache to cause spillover. + Map segmentSizeTracker = new HashMap<>(); for (int iter = 0; iter < numOfItems1; iter++) { ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + int keySegment = tieredSpilloverCache.getSegmentNumber(key); LoadAwareCacheLoader, String> loadAwareCacheLoader = getLoadAwareCacheLoader(); tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader); + if (segmentSizeTracker.get(keySegment) == null) { + segmentSizeTracker.put(keySegment, Integer.valueOf(1)); + } else { + Integer updatedValue = segmentSizeTracker.get(keySegment) + 1; + segmentSizeTracker.put(keySegment, updatedValue); + } + } + int expectedEvictions = 0; + for (int i = 0; i < numberOfSegments; i++) { + if (segmentSizeTracker.get(i) == null) { + continue; + } + if (segmentSizeTracker.get(i) > perSegmentEntryCapacity.get(i)) { + expectedEvictions += segmentSizeTracker.get(i) - perSegmentEntryCapacity.get(i); + } } - ICache onHeapCache = tieredSpilloverCache.getOnHeapCache(); - ICache diskCache = tieredSpilloverCache.getDiskCache(); List> diskCacheKeys = new ArrayList<>(); - tieredSpilloverCache.getDiskCache().keys().forEach(diskCacheKeys::add); - long actualDiskCacheCount = diskCache.count(); + tieredSpilloverCache.getDiskCacheKeys().forEach(diskCacheKeys::add); + long actualDiskCacheCount = tieredSpilloverCache.diskCacheCount(); long actualTieredCacheCount = tieredSpilloverCache.count(); - assertEquals(onHeapCacheSize, onHeapCache.count()); - assertEquals(numOfItems1 - onHeapCacheSize, actualDiskCacheCount); - assertEquals(0, removalListener.evictionsMetric.count()); - assertEquals(numOfItems1, actualTieredCacheCount); + assertEquals(getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP), tieredSpilloverCache.onHeapCacheCount()); + assertEquals(getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK), actualDiskCacheCount); + assertEquals(expectedEvictions, removalListener.evictionsMetric.count()); + assertEquals( + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP) + getItemsForTier( + tieredSpilloverCache, + TIER_DIMENSION_VALUE_DISK + ), + actualTieredCacheCount + ); for (ICacheKey diskKey : diskCacheKeys) { assertNotNull(tieredSpilloverCache.get(diskKey)); } - tieredSpilloverCache.enableDisableDiskCache(false); // Disable disk cache now. int numOfItems2 = totalSize - numOfItems1; for (int iter = 0; iter < numOfItems2; iter++) { ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + int keySegment = tieredSpilloverCache.getSegmentNumber(key); + TieredSpilloverCache.TieredSpilloverCacheSegment segment = + tieredSpilloverCache.tieredSpilloverCacheSegments[keySegment]; LoadAwareCacheLoader, String> loadAwareCacheLoader = getLoadAwareCacheLoader(); tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader); } @@ -1407,20 +1668,21 @@ public void testGetPutAndInvalidateWithDiskCacheDisabled() throws Exception { assertNull(tieredSpilloverCache.get(diskKey)); // Considering disk cache is disabled, we shouldn't find // these keys. } - assertEquals(onHeapCacheSize, onHeapCache.count()); // Should remain same. - assertEquals(0, diskCache.count() - actualDiskCacheCount); // Considering it is disabled now, shouldn't cache + assertEquals(getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP), tieredSpilloverCache.onHeapCacheCount()); // Should + // remain + // same. + assertEquals(0, tieredSpilloverCache.diskCacheCount() - actualDiskCacheCount); // Considering it is disabled now, shouldn't + // cache // any more items. - assertEquals(numOfItems2, removalListener.evictionsMetric.count()); // Considering onHeap cache was already - // full, we should all existing onHeap entries being evicted. - assertEquals(0, tieredSpilloverCache.count() - actualTieredCacheCount); // Count still returns disk cache - // entries count as they haven't been cleared yet. + assertTrue(removalListener.evictionsMetric.count() > 0); + // Considering onHeap cache was already full, we should have some onHeap entries being evicted. long lastKnownTieredCacheEntriesCount = tieredSpilloverCache.count(); // Clear up disk cache keys. for (ICacheKey diskKey : diskCacheKeys) { tieredSpilloverCache.invalidate(diskKey); } - assertEquals(0, diskCache.count()); + assertEquals(0, tieredSpilloverCache.diskCacheCount()); assertEquals(lastKnownTieredCacheEntriesCount - diskCacheKeys.size(), tieredSpilloverCache.count()); tieredSpilloverCache.invalidateAll(); // Clear up all the keys. @@ -1438,13 +1700,14 @@ public void testTiersDoNotTrackStats() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + 1 ); // do some gets to put entries in both tiers @@ -1455,8 +1718,11 @@ public void testTiersDoNotTrackStats() throws Exception { tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader); } assertNotEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.stats().getTotalStats()); - assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.getOnHeapCache().stats().getTotalStats()); - ImmutableCacheStats diskStats = tieredSpilloverCache.getDiskCache().stats().getTotalStats(); + assertEquals( + new ImmutableCacheStats(0, 0, 0, 0, 0), + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getOnHeapCache().stats().getTotalStats() + ); + ImmutableCacheStats diskStats = tieredSpilloverCache.tieredSpilloverCacheSegments[0].getDiskCache().stats().getTotalStats(); assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), diskStats); } @@ -1469,8 +1735,9 @@ public void testTierStatsAddCorrectly() throws Exception { * totalEntries = heapEntries + diskEntries */ - int onHeapCacheSize = randomIntBetween(10, 30); - int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100); + int onHeapCacheSize = randomIntBetween(300, 600); + int diskCacheSize = randomIntBetween(700, 1200); + int numberOfSegments = getNumberOfSegments(); int keyValueSize = 50; MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( @@ -1479,13 +1746,14 @@ public void testTierStatsAddCorrectly() throws Exception { removalListener, Settings.builder() .put( - OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) - .get(MAXIMUM_SIZE_IN_BYTES_KEY) - .getKey(), + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), onHeapCacheSize * keyValueSize + "b" ) .build(), - 0 + 0, + numberOfSegments ); List> usedKeys = new ArrayList<>(); @@ -1529,16 +1797,28 @@ public void testTierStatsAddCorrectly() throws Exception { clusterSettings.applySettings( Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false).build() ); - - int newMisses = randomIntBetween(10, 30); - for (int i = 0; i < newMisses; i++) { + Map onHeapExpectedSize = getSegmentOnHeapCacheSize( + numberOfSegments, + onHeapCacheSize * keyValueSize, + keyValueSize + ); + int numOfItems = randomIntBetween(10, 30); + int newMisses = 0; + for (int i = 0; i < numOfItems; i++) { LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); - tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader); + ICacheKey iCacheKey = getICacheKey(UUID.randomUUID().toString()); + int keySegment = tieredSpilloverCache.getSegmentNumber(iCacheKey); + if (tieredSpilloverCache.tieredSpilloverCacheSegments[keySegment].getOnHeapCache().count() >= onHeapExpectedSize.get( + keySegment + )) { + newMisses++; + } + tieredSpilloverCache.computeIfAbsent(iCacheKey, tieredCacheLoader); } totalStats = tieredSpilloverCache.stats().getTotalStats(); heapStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP); - assertEquals(missesBeforeDisablingDiskCache + newMisses, totalStats.getMisses()); + assertEquals(missesBeforeDisablingDiskCache + numOfItems, totalStats.getMisses()); assertEquals(heapTierEvictionsBeforeDisablingDiskCache + newMisses, heapStats.getEvictions()); assertEquals(evictionsBeforeDisablingDiskCache + newMisses, totalStats.getEvictions()); @@ -1546,7 +1826,290 @@ public void testTierStatsAddCorrectly() throws Exception { clusterSettings.applySettings( Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), true).build() ); + } + + public void testPutForAKeyWhichAlreadyExists() { + int onHeapCacheSize = 1; + int diskCacheSize = 3; + int keyValueSize = 1; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(), + 0, + 1 + ); + ICacheKey key1 = getICacheKey("key1"); + ICacheKey key2 = getICacheKey("key2"); + tieredSpilloverCache.put(key1, "key1"); // Goes to onHeap cache. + tieredSpilloverCache.put(key2, "key2"); // Goes to onHeap cache. And key1 evicted to disk cache. + List> diskKeys = new ArrayList<>(); + List> onHeapCacheKeys = new ArrayList<>(); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getDiskCache().keys().forEach(diskKeys::add); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getOnHeapCache().keys().forEach(onHeapCacheKeys::add); + assertEquals(1, onHeapCacheKeys.size()); + assertEquals(1, diskKeys.size()); + assertTrue(onHeapCacheKeys.contains(key2)); + assertTrue(diskKeys.contains(key1)); + assertEquals("key1", tieredSpilloverCache.get(key1)); + + // Now try to put key1 again onto tiered cache with new value. + tieredSpilloverCache.put(key1, "dummy"); + diskKeys.clear(); + onHeapCacheKeys.clear(); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getDiskCache().keys().forEach(diskKeys::add); + tieredSpilloverCache.tieredSpilloverCacheSegments[0].getOnHeapCache().keys().forEach(onHeapCacheKeys::add); + assertEquals(1, onHeapCacheKeys.size()); + assertEquals(1, diskKeys.size()); + assertTrue(onHeapCacheKeys.contains(key2)); + assertTrue(diskKeys.contains(key1)); + assertEquals("dummy", tieredSpilloverCache.get(key1)); + } + public void testTieredCacheThrowingExceptionOnPerSegmentSizeBeingZero() { + int onHeapCacheSize = 10; + int diskCacheSize = randomIntBetween(700, 1200); + int keyValueSize = 1; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + assertThrows( + IllegalArgumentException.class, + () -> initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(), + 0, + 256 + ) + ); + } + + public void testTieredCacheWithZeroNumberOfSegments() { + int onHeapCacheSize = 10; + int diskCacheSize = randomIntBetween(700, 1200); + int keyValueSize = 1; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + assertThrows( + ZERO_SEGMENT_COUNT_EXCEPTION_MESSAGE, + IllegalArgumentException.class, + () -> initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(), + 0, + 0 + ) + ); + } + + public void testWithInvalidSegmentNumber() throws Exception { + int onHeapCacheSize = 10; + int diskCacheSize = randomIntBetween(700, 1200); + int keyValueSize = 1; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + MockDiskCache.MockDiskCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 1) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 3) + .build(); + String storagePath = getStoragePath(settings); + assertThrows( + INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE, + IllegalArgumentException.class, + () -> new TieredSpilloverCache.TieredSpilloverCacheFactory().create( + new CacheConfig.Builder().setKeyType(String.class) + .setKeyType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setRemovalListener(removalListener) + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setSettings(settings) + .setDimensionNames(dimensionNames) + .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken + // 20_000_000 ns = 20 ms to compute + .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) + .build(), + CacheType.INDICES_REQUEST_CACHE, + Map.of( + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, + new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), + MockDiskCache.MockDiskCacheFactory.NAME, + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) + ) + ) + ); + } + + public void testWithVeryLowDiskCacheSize() throws Exception { + int onHeapCacheSize = 10; + int diskCacheSize = randomIntBetween(700, 1200); + int keyValueSize = 1; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + MockDiskCache.MockDiskCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + 1L + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put(TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()).getKey(), 2) + .build(); + String storagePath = getStoragePath(settings); + assertThrows( + IllegalArgumentException.class, + () -> new TieredSpilloverCache.TieredSpilloverCacheFactory().create( + new CacheConfig.Builder().setKeyType(String.class) + .setKeyType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setRemovalListener(removalListener) + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setSettings(settings) + .setDimensionNames(dimensionNames) + .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken + // 20_000_000 ns = 20 ms to compute + .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) + .build(), + CacheType.INDICES_REQUEST_CACHE, + Map.of( + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, + new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), + MockDiskCache.MockDiskCacheFactory.NAME, + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) + ) + ) + ); + } + + public void testTieredCacheDefaultSegmentCount() { + int onHeapCacheSize = 500; + int keyValueSize = 1; + + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + MockDiskCache.MockDiskCacheFactory.NAME + ) + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .build(); + String storagePath = getStoragePath(settings); + + TieredSpilloverCache tieredSpilloverCache = (TieredSpilloverCache< + String, + String>) new TieredSpilloverCache.TieredSpilloverCacheFactory().create( + new CacheConfig.Builder().setKeyType(String.class) + .setKeyType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setRemovalListener(removalListener) + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setSettings(settings) + .setDimensionNames(dimensionNames) + .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken + // 20_000_000 ns = 20 ms to compute + .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) + .build(), + CacheType.INDICES_REQUEST_CACHE, + Map.of( + OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME, + new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(), + MockDiskCache.MockDiskCacheFactory.NAME, + new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false, keyValueSize) + ) + ); + assertEquals(TieredSpilloverCacheSettings.defaultSegments(), tieredSpilloverCache.getNumberOfSegments()); + assertTrue(VALID_SEGMENT_COUNT_VALUES.contains(tieredSpilloverCache.getNumberOfSegments())); } private List getMockDimensions() { @@ -1631,7 +2194,10 @@ private TieredSpilloverCache getTieredSpilloverCache( ICache.Factory mockDiskCacheFactory, CacheConfig cacheConfig, List> policies, - RemovalListener, String> removalListener + RemovalListener, String> removalListener, + int numberOfSegments, + long onHeapCacheSizeInBytes, + long diskCacheSize ) { TieredSpilloverCache.Builder builder = new TieredSpilloverCache.Builder().setCacheType( CacheType.INDICES_REQUEST_CACHE @@ -1639,6 +2205,9 @@ private TieredSpilloverCache getTieredSpilloverCache( .setRemovalListener(removalListener) .setOnHeapCacheFactory(onHeapCacheFactory) .setDiskCacheFactory(mockDiskCacheFactory) + .setNumberOfSegments(numberOfSegments) + .setDiskCacheSize(diskCacheSize) + .setOnHeapCacheSizeInBytes(onHeapCacheSizeInBytes) .setCacheConfig(cacheConfig); if (policies != null) { builder.addPolicies(policies); @@ -1654,7 +2223,27 @@ private TieredSpilloverCache initializeTieredSpilloverCache( long diskDeliberateDelay ) { - return intializeTieredSpilloverCache(keyValueSize, diskCacheSize, removalListener, settings, diskDeliberateDelay, null); + return intializeTieredSpilloverCache(keyValueSize, diskCacheSize, removalListener, settings, diskDeliberateDelay, null, 256); + } + + private TieredSpilloverCache initializeTieredSpilloverCache( + int keyValueSize, + int diskCacheSize, + RemovalListener, String> removalListener, + Settings settings, + long diskDeliberateDelay, + int numberOfSegments + + ) { + return intializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + settings, + diskDeliberateDelay, + null, + numberOfSegments + ); } private TieredSpilloverCache intializeTieredSpilloverCache( @@ -1663,9 +2252,19 @@ private TieredSpilloverCache intializeTieredSpilloverCache( RemovalListener, String> removalListener, Settings settings, long diskDeliberateDelay, - List> policies + List> policies, + int numberOfSegments ) { ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); + String storagePath; + try (NodeEnvironment environment = newNodeEnvironment(settings)) { + storagePath = environment.nodePaths()[0].path + "/test"; + } catch (IOException e) { + throw new OpenSearchException("Exception occurred", e); + } + long onHeapCacheSizeInBytes = TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).get(settings).getBytes(); CacheConfig cacheConfig = new CacheConfig.Builder().setKeyType(String.class) .setKeyType(String.class) .setWeigher((k, v) -> keyValueSize) @@ -1685,17 +2284,39 @@ private TieredSpilloverCache intializeTieredSpilloverCache( .build() ) .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) .build(); - ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize, false); + ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory( + diskDeliberateDelay, + diskCacheSize, + false, + keyValueSize + ); - return getTieredSpilloverCache(onHeapCacheFactory, mockDiskCacheFactory, cacheConfig, policies, removalListener); + return getTieredSpilloverCache( + onHeapCacheFactory, + mockDiskCacheFactory, + cacheConfig, + policies, + removalListener, + numberOfSegments, + onHeapCacheSizeInBytes, + diskCacheSize + ); } private CacheConfig getCacheConfig( int keyValueSize, Settings settings, - RemovalListener, String> removalListener + RemovalListener, String> removalListener, + int numberOfSegments ) { + String storagePath; + try (NodeEnvironment environment = newNodeEnvironment(settings)) { + storagePath = environment.nodePaths()[0].path + "/test"; + } catch (IOException e) { + throw new OpenSearchException("Exception occurred", e); + } return new CacheConfig.Builder().setKeyType(String.class) .setKeyType(String.class) .setWeigher((k, v) -> keyValueSize) @@ -1715,6 +2336,8 @@ private CacheConfig getCacheConfig( .build() ) .setClusterSettings(clusterSettings) + .setStoragePath(storagePath) + .setSegmentCount(numberOfSegments) .build(); } @@ -1756,6 +2379,16 @@ private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache t return snapshot; } + private String getStoragePath(Settings settings) { + String storagePath; + try (NodeEnvironment environment = newNodeEnvironment(settings)) { + storagePath = environment.nodePaths()[0].path + "/test"; + } catch (IOException e) { + throw new OpenSearchException("Exception occurred", e); + } + return storagePath; + } + private void verifyComputeIfAbsentThrowsException( Class expectedException, LoadAwareCacheLoader, String> loader, @@ -1780,7 +2413,8 @@ private void verifyComputeIfAbsentThrowsException( diskCacheSize, removalListener, settings, - 0 + 0, + 1 ); int numberOfSameKeys = randomIntBetween(10, onHeapCacheSize - 1); @@ -1816,6 +2450,29 @@ private void verifyComputeIfAbsentThrowsException( assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); } + private int getNumberOfSegments() { + return randomFrom(2, 4, 8, 16, 32, 64, 128, 256); + } + + private Map getSegmentOnHeapCacheSize(int numberOfSegments, int onHeapCacheSizeInBytes, int keyValueSize) { + Map expectedSegmentOnHeapCacheSize = new HashMap<>(); + for (int i = 0; i < numberOfSegments; i++) { + int perSegmentOnHeapCacheSizeBytes = onHeapCacheSizeInBytes / numberOfSegments; + int perSegmentOnHeapCacheEntries = perSegmentOnHeapCacheSizeBytes / keyValueSize; + expectedSegmentOnHeapCacheSize.put(i, perSegmentOnHeapCacheEntries); + } + return expectedSegmentOnHeapCacheSize; + } + + private Map getSegmentMockDiskCacheSize(int numberOfSegments, int diskCacheSize) { + Map expectedSegmentDiskCacheSize = new HashMap<>(); + for (int i = 0; i < numberOfSegments; i++) { + int perSegmentDiskCacheEntries = diskCacheSize / numberOfSegments; + expectedSegmentDiskCacheSize.put(i, perSegmentDiskCacheEntries); + } + return expectedSegmentDiskCacheSize; + } + private ImmutableCacheStats getTotalStatsSnapshot(TieredSpilloverCache tsc) throws IOException { ImmutableCacheStatsHolder cacheStats = tsc.stats(new String[0]); return cacheStats.getStatsForDimensionValues(List.of()); diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java index d173155080f6a..cbc104f2d0b00 100644 --- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java @@ -88,7 +88,7 @@ public class EhcacheDiskCacheSettings { */ public static final Setting.AffixSetting DISK_CACHE_ALIAS_SETTING = Setting.suffixKeySetting( EhcacheDiskCache.EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME + ".alias", - (key) -> Setting.simpleString(key, "", NodeScope) + (key) -> Setting.simpleString(key, "ehcache_disk", NodeScope) ); /** diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java index 4a95b04de3952..0fa0f8162bb98 100644 --- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java @@ -101,14 +101,18 @@ public class EhcacheDiskCache implements ICache { private static final Logger logger = LogManager.getLogger(EhcacheDiskCache.class); // Unique id associated with this cache. - private final static String UNIQUE_ID = UUID.randomUUID().toString(); - private final static String THREAD_POOL_ALIAS_PREFIX = "ehcachePool"; + final static String UNIQUE_ID = UUID.randomUUID().toString(); + final static String THREAD_POOL_ALIAS_PREFIX = "ehcachePool"; + final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB + final static String CACHE_DATA_CLEANUP_DURING_INITIALIZATION_EXCEPTION = "Failed to delete ehcache disk cache under " + + "path: %s during initialization. Please clean this up manually and restart the process"; + // A Cache manager can create many caches. private final PersistentCacheManager cacheManager; // Disk cache. Using ByteArrayWrapper to compare two byte[] by values rather than the default reference checks @SuppressWarnings({ "rawtypes" }) // We have to use the raw type as there's no way to pass the "generic class" to ehcache - private Cache cache; + private final Cache cache; private final long maxWeightInBytes; private final String storagePath; private final Class keyType; @@ -124,10 +128,6 @@ public class EhcacheDiskCache implements ICache { private final Serializer keySerializer; private final Serializer valueSerializer; - final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB - final static String CACHE_DATA_CLEANUP_DURING_INITIALIZATION_EXCEPTION = "Failed to delete ehcache disk cache under " - + "path: %s during initialization. Please clean this up manually and restart the process"; - /** * Used in computeIfAbsent to synchronize loading of a given key. This is needed as ehcache doesn't provide a * computeIfAbsent method. @@ -199,6 +199,12 @@ private Cache buildCache(Duration expireAfterAccess // Creating the cache requires permissions specified in plugin-security.policy return AccessController.doPrivileged((PrivilegedAction>) () -> { try { + int segmentCount = (Integer) EhcacheDiskCacheSettings.getSettingListForCacheType(cacheType) + .get(DISK_SEGMENT_KEY) + .get(settings); + if (builder.getNumberOfSegments() > 0) { + segmentCount = builder.getNumberOfSegments(); + } return this.cacheManager.createCache( this.diskCacheAlias, CacheConfigurationBuilder.newCacheConfigurationBuilder( @@ -232,7 +238,7 @@ public Duration getExpiryForUpdate( (Integer) EhcacheDiskCacheSettings.getSettingListForCacheType(cacheType) .get(DISK_WRITE_CONCURRENCY_KEY) .get(settings), - (Integer) EhcacheDiskCacheSettings.getSettingListForCacheType(cacheType).get(DISK_SEGMENT_KEY).get(settings) + segmentCount ) ) .withKeySerializer(new KeySerializerWrapper(keySerializer)) @@ -709,8 +715,19 @@ public ICache create(CacheConfig config, CacheType cacheType, throw new IllegalArgumentException("EhcacheDiskCache requires a value serializer of type Serializer"); } - return new Builder().setStoragePath((String) settingList.get(DISK_STORAGE_PATH_KEY).get(settings)) - .setDiskCacheAlias((String) settingList.get(DISK_CACHE_ALIAS_KEY).get(settings)) + String storagePath = (String) settingList.get(DISK_STORAGE_PATH_KEY).get(settings); + // If we read the storage path directly from the setting, we have to add the segment number at the end. + if (storagePath == null || storagePath.isBlank()) { + // In case storage path is not explicitly set by user, use default path. + // Since this comes from the TSC, it already has the segment number at the end. + storagePath = config.getStoragePath(); + } + String diskCacheAlias = (String) settingList.get(DISK_CACHE_ALIAS_KEY).get(settings); + if (config.getCacheAlias() != null && !config.getCacheAlias().isBlank()) { + diskCacheAlias = config.getCacheAlias(); + } + EhcacheDiskCache.Builder builder = (Builder) new Builder().setStoragePath(storagePath) + .setDiskCacheAlias(diskCacheAlias) .setIsEventListenerModeSync((Boolean) settingList.get(DISK_LISTENER_MODE_SYNC_KEY).get(settings)) .setCacheType(cacheType) .setKeyType((config.getKeyType())) @@ -721,9 +738,21 @@ public ICache create(CacheConfig config, CacheType cacheType, .setWeigher(config.getWeigher()) .setRemovalListener(config.getRemovalListener()) .setExpireAfterAccess((TimeValue) settingList.get(DISK_CACHE_EXPIRE_AFTER_ACCESS_KEY).get(settings)) - .setMaximumWeightInBytes((Long) settingList.get(DISK_MAX_SIZE_IN_BYTES_KEY).get(settings)) - .setSettings(settings) - .build(); + .setSettings(settings); + long maxSizeInBytes = (Long) settingList.get(DISK_MAX_SIZE_IN_BYTES_KEY).get(settings); + // If config value is set, use this instead. + if (config.getMaxSizeInBytes() > 0) { + builder.setMaximumWeightInBytes(config.getMaxSizeInBytes()); + } else { + builder.setMaximumWeightInBytes(maxSizeInBytes); + } + int segmentCount = (Integer) EhcacheDiskCacheSettings.getSettingListForCacheType(cacheType).get(DISK_SEGMENT_KEY).get(settings); + if (config.getSegmentCount() > 0) { + builder.setNumberOfSegments(config.getSegmentCount()); + } else { + builder.setNumberOfSegments(segmentCount); + } + return builder.build(); } @Override diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java index 2bc24227bb513..a0d0aa4ec4914 100644 --- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java @@ -433,6 +433,74 @@ public String load(ICacheKey key) { } } + public void testComputeIfAbsentConcurrentlyWithMultipleEhcacheDiskCache() throws IOException { + Settings settings = Settings.builder().build(); + MockRemovalListener removalListener = new MockRemovalListener<>(); + List> iCaches = new ArrayList<>(); + int segments = 4; + try (NodeEnvironment env = newNodeEnvironment(settings)) { + ICache.Factory ehcacheFactory = new EhcacheDiskCache.EhcacheDiskCacheFactory(); + for (int i = 1; i <= segments; i++) { + ICache ehcacheTest = ehcacheFactory.create( + new CacheConfig.Builder().setValueType(String.class) + .setKeyType(String.class) + .setRemovalListener(removalListener) + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setDimensionNames(List.of(dimensionName)) + .setWeigher(getWeigher()) + .setMaxSizeInBytes(CACHE_SIZE_IN_BYTES * 100) + .setSettings( + Settings.builder() + .put( + EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(DISK_MAX_SIZE_IN_BYTES_KEY) + .getKey(), + CACHE_SIZE_IN_BYTES + ) + .put( + EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(DISK_STORAGE_PATH_KEY) + .getKey(), + env.nodePaths()[0].indicesPath.toString() + "/request_cache/" + i + ) + .put( + EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(DISK_LISTENER_MODE_SYNC_KEY) + .getKey(), + true + ) + .build() + ) + .build(), + CacheType.INDICES_REQUEST_CACHE, + Map.of() + ); + iCaches.add(ehcacheTest); + } + int randomKeys = randomIntBetween(100, 300); + Map, String> keyValueMap = new HashMap<>(); + for (int i = 0; i < randomKeys; i++) { + keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString()); + } + for (Map.Entry, String> entry : keyValueMap.entrySet()) { + ICache ehcacheTest = iCaches.get(entry.getKey().hashCode() & (segments - 1)); + ehcacheTest.put(entry.getKey(), entry.getValue()); + } + for (Map.Entry, String> entry : keyValueMap.entrySet()) { + ICache ehcacheTest = iCaches.get(entry.getKey().hashCode() & (segments - 1)); + String value = ehcacheTest.get(entry.getKey()); + assertEquals(entry.getValue(), value); + } + int count = 0; + for (int i = 0; i < segments; i++) { + count += iCaches.get(i).count(); + iCaches.get(i).close(); + } + assertEquals(randomKeys, count); + } + } + public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception { Settings settings = Settings.builder().build(); MockRemovalListener removalListener = new MockRemovalListener<>(); diff --git a/server/src/main/java/org/opensearch/common/cache/Cache.java b/server/src/main/java/org/opensearch/common/cache/Cache.java index caae81e4387b4..da683bfff088d 100644 --- a/server/src/main/java/org/opensearch/common/cache/Cache.java +++ b/server/src/main/java/org/opensearch/common/cache/Cache.java @@ -115,8 +115,20 @@ public class Cache { // the removal callback private RemovalListener removalListener = notification -> {}; - // use CacheBuilder to construct - Cache() {} + private final int numberOfSegments; + private static final int NUMBER_OF_SEGMENTS_DEFAULT = 256; + + Cache(final int numberOfSegments) { + if (numberOfSegments != -1) { + this.numberOfSegments = numberOfSegments; + } else { + this.numberOfSegments = NUMBER_OF_SEGMENTS_DEFAULT; + } + this.segments = new CacheSegment[this.numberOfSegments]; + for (int i = 0; i < this.numberOfSegments; i++) { + this.segments[i] = new CacheSegment<>(); + } + } void setExpireAfterAccessNanos(long expireAfterAccessNanos) { if (expireAfterAccessNanos <= 0) { @@ -366,15 +378,8 @@ void eviction() { } } - public static final int NUMBER_OF_SEGMENTS = 256; @SuppressWarnings("unchecked") - private final CacheSegment[] segments = new CacheSegment[NUMBER_OF_SEGMENTS]; - - { - for (int i = 0; i < segments.length; i++) { - segments[i] = new CacheSegment<>(); - } - } + private final CacheSegment[] segments; Entry head; Entry tail; @@ -382,6 +387,10 @@ void eviction() { // lock protecting mutations to the LRU list private final ReleasableLock lruLock = new ReleasableLock(new ReentrantLock()); + int getNumberOfSegments() { + return numberOfSegments; + } + /** * Returns the value to which the specified key is mapped, or null if this map contains no mapping for the key. * @@ -588,9 +597,9 @@ public void invalidate(K key, V value) { public void invalidateAll() { Entry h; - boolean[] haveSegmentLock = new boolean[NUMBER_OF_SEGMENTS]; + boolean[] haveSegmentLock = new boolean[this.numberOfSegments]; try { - for (int i = 0; i < NUMBER_OF_SEGMENTS; i++) { + for (int i = 0; i < this.numberOfSegments; i++) { segments[i].segmentLock.writeLock().lock(); haveSegmentLock[i] = true; } @@ -607,7 +616,7 @@ public void invalidateAll() { weight = 0; } } finally { - for (int i = NUMBER_OF_SEGMENTS - 1; i >= 0; i--) { + for (int i = this.numberOfSegments - 1; i >= 0; i--) { if (haveSegmentLock[i]) { segments[i].segmentLock.writeLock().unlock(); } @@ -940,6 +949,6 @@ public ToLongBiFunction getWeigher() { } private CacheSegment getCacheSegment(K key) { - return segments[key.hashCode() & 0xff]; + return segments[key.hashCode() & (this.numberOfSegments - 1)]; } } diff --git a/server/src/main/java/org/opensearch/common/cache/CacheBuilder.java b/server/src/main/java/org/opensearch/common/cache/CacheBuilder.java index b6d7979aa4108..8f93e3358a375 100644 --- a/server/src/main/java/org/opensearch/common/cache/CacheBuilder.java +++ b/server/src/main/java/org/opensearch/common/cache/CacheBuilder.java @@ -34,9 +34,13 @@ import org.opensearch.common.unit.TimeValue; +import java.util.Locale; import java.util.Objects; import java.util.function.ToLongBiFunction; +import static org.opensearch.common.cache.settings.CacheSettings.INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE; +import static org.opensearch.common.cache.settings.CacheSettings.VALID_SEGMENT_COUNT_VALUES; + /** * The cache builder. * @@ -48,6 +52,7 @@ public class CacheBuilder { private long expireAfterWriteNanos = -1; private ToLongBiFunction weigher; private RemovalListener removalListener; + private int numberOfSegments = -1; public static CacheBuilder builder() { return new CacheBuilder<>(); @@ -55,6 +60,14 @@ public static CacheBuilder builder() { private CacheBuilder() {} + public CacheBuilder setNumberOfSegments(int numberOfSegments) { + if (!VALID_SEGMENT_COUNT_VALUES.contains(numberOfSegments)) { + throw new IllegalArgumentException(String.format(Locale.ROOT, INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE, "Cache")); + } + this.numberOfSegments = numberOfSegments; + return this; + } + public CacheBuilder setMaximumWeight(long maximumWeight) { if (maximumWeight < 0) { throw new IllegalArgumentException("maximumWeight < 0"); @@ -108,7 +121,7 @@ public CacheBuilder removalListener(RemovalListener removalListener) } public Cache build() { - Cache cache = new Cache<>(); + Cache cache = new Cache<>(numberOfSegments); if (maximumWeight != -1) { cache.setMaximumWeight(maximumWeight); } diff --git a/server/src/main/java/org/opensearch/common/cache/settings/CacheSettings.java b/server/src/main/java/org/opensearch/common/cache/settings/CacheSettings.java index 43a047f0f22c6..7a810d92e3d0d 100644 --- a/server/src/main/java/org/opensearch/common/cache/settings/CacheSettings.java +++ b/server/src/main/java/org/opensearch/common/cache/settings/CacheSettings.java @@ -12,12 +12,25 @@ import org.opensearch.common.cache.CacheType; import org.opensearch.common.settings.Setting; +import java.util.List; + /** * Settings related to cache. */ @ExperimentalApi public class CacheSettings { + /** + * Only includes values which is power of 2 as we use bitwise logic: (key AND (segmentCount -1)) to calculate + * segmentNumber which works well only with such values. Stores it in sorted order. + */ + public static final List VALID_SEGMENT_COUNT_VALUES = List.of(1, 2, 4, 8, 16, 32, 64, 128, 256); + + /** + * Exception message for invalid segment number. + */ + public static final String INVALID_SEGMENT_COUNT_EXCEPTION_MESSAGE = "Cache: %s segment count should be power of two up-to 256"; + /** * Used to store cache store name for desired cache types within OpenSearch. * Setting pattern: {cache_type}.store.name diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java index 569653bec2a3d..571383a9fce6a 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java @@ -54,8 +54,10 @@ public class OpenSearchOnHeapCache implements ICache, RemovalListene private final List dimensionNames; private final ToLongBiFunction, V> weigher; private final boolean statsTrackingEnabled; + private final long maximumWeight; public OpenSearchOnHeapCache(Builder builder) { + this.maximumWeight = builder.getMaxWeightInBytes(); CacheBuilder, V> cacheBuilder = CacheBuilder., V>builder() .setMaximumWeight(builder.getMaxWeightInBytes()) .weigher(builder.getWeigher()) @@ -63,6 +65,9 @@ public OpenSearchOnHeapCache(Builder builder) { if (builder.getExpireAfterAcess() != null) { cacheBuilder.setExpireAfterAccess(builder.getExpireAfterAcess()); } + if (builder.getNumberOfSegments() > 0) { + cacheBuilder.setNumberOfSegments(builder.getNumberOfSegments()); + } cache = cacheBuilder.build(); this.dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null"); this.statsTrackingEnabled = builder.getStatsTrackingEnabled(); @@ -75,6 +80,11 @@ public OpenSearchOnHeapCache(Builder builder) { this.weigher = builder.getWeigher(); } + // package private for testing + long getMaximumWeight() { + return this.maximumWeight; + } + @Override public V get(ICacheKey key) { V value = cache.get(key); @@ -174,18 +184,33 @@ public ICache create(CacheConfig config, CacheType cacheType, boolean statsTrackingEnabled = statsTrackingEnabled(config.getSettings(), config.getStatsTrackingEnabled()); ICacheBuilder builder = new Builder().setDimensionNames(config.getDimensionNames()) .setStatsTrackingEnabled(statsTrackingEnabled) - .setMaximumWeightInBytes(((ByteSizeValue) settingList.get(MAXIMUM_SIZE_IN_BYTES_KEY).get(settings)).getBytes()) .setExpireAfterAccess(((TimeValue) settingList.get(EXPIRE_AFTER_ACCESS_KEY).get(settings))) .setWeigher(config.getWeigher()) .setRemovalListener(config.getRemovalListener()); Setting cacheSettingForCacheType = CacheSettings.CACHE_TYPE_STORE_NAME.getConcreteSettingForNamespace( cacheType.getSettingPrefix() ); + long maxSizeInBytes = ((ByteSizeValue) settingList.get(MAXIMUM_SIZE_IN_BYTES_KEY).get(settings)).getBytes(); + + if (config.getMaxSizeInBytes() > 0) { // If this is passed from upstream(like tieredCache), then use this + // instead. + builder.setMaximumWeightInBytes(config.getMaxSizeInBytes()); + } else { + builder.setMaximumWeightInBytes(maxSizeInBytes); + } + if (config.getSegmentCount() > 0) { + builder.setNumberOfSegments(config.getSegmentCount()); + } else { + builder.setNumberOfSegments(-1); // By default it will use 256 segments. + } + String storeName = cacheSettingForCacheType.get(settings); if (!FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings) || (storeName == null || storeName.isBlank())) { // For backward compatibility as the user intent is to use older settings. builder.setMaximumWeightInBytes(config.getMaxSizeInBytes()); builder.setExpireAfterAccess(config.getExpireAfterAccess()); + builder.setNumberOfSegments(-1); // By default it will use 256 as we don't want to use this setting + // when user wants to use older default onHeap cache settings. } return builder.build(); } diff --git a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java index a308d1db88258..d35e22ffdc978 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java +++ b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java @@ -39,6 +39,8 @@ public abstract class ICacheBuilder { private boolean statsTrackingEnabled = true; + private int numberOfSegments; + public ICacheBuilder() {} public ICacheBuilder setMaximumWeightInBytes(long sizeInBytes) { @@ -71,6 +73,11 @@ public ICacheBuilder setStatsTrackingEnabled(boolean statsTrackingEnabled) return this; } + public ICacheBuilder setNumberOfSegments(int numberOfSegments) { + this.numberOfSegments = numberOfSegments; + return this; + } + public long getMaxWeightInBytes() { return maxWeightInBytes; } @@ -79,6 +86,10 @@ public TimeValue getExpireAfterAcess() { return expireAfterAcess; } + public int getNumberOfSegments() { + return numberOfSegments; + } + public ToLongBiFunction, V> getWeigher() { return weigher; } diff --git a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java index 0c54ac57a9b18..ddad416c251a9 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java +++ b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java @@ -70,6 +70,12 @@ public class CacheConfig { private final boolean statsTrackingEnabled; + private final String storagePath; + + private final int segmentCount; + + private final String cacheAlias; + private CacheConfig(Builder builder) { this.keyType = builder.keyType; this.valueType = builder.valueType; @@ -84,6 +90,9 @@ private CacheConfig(Builder builder) { this.expireAfterAccess = builder.expireAfterAccess; this.clusterSettings = builder.clusterSettings; this.statsTrackingEnabled = builder.statsTrackingEnabled; + this.storagePath = builder.storagePath; + this.segmentCount = builder.segmentCount; + this.cacheAlias = builder.cacheAlias; } public Class getKeyType() { @@ -138,6 +147,18 @@ public boolean getStatsTrackingEnabled() { return statsTrackingEnabled; } + public String getStoragePath() { + return storagePath; + } + + public int getSegmentCount() { + return segmentCount; + } + + public String getCacheAlias() { + return cacheAlias; + } + /** * Builder class to build Cache config related parameters. * @param Type of key. @@ -163,6 +184,9 @@ public static class Builder { private TimeValue expireAfterAccess; private ClusterSettings clusterSettings; private boolean statsTrackingEnabled = true; + private String storagePath; + private int segmentCount; + private String cacheAlias; public Builder() {} @@ -231,6 +255,21 @@ public Builder setStatsTrackingEnabled(boolean statsTrackingEnabled) { return this; } + public Builder setStoragePath(String storagePath) { + this.storagePath = storagePath; + return this; + } + + public Builder setSegmentCount(int segmentCount) { + this.segmentCount = segmentCount; + return this; + } + + public Builder setCacheAlias(String cacheAlias) { + this.cacheAlias = cacheAlias; + return this; + } + public CacheConfig build() { return new CacheConfig<>(this); } diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index 71f8cf5a78ec5..156fe32ff5809 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -68,6 +68,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.env.NodeEnvironment; import org.opensearch.index.shard.IndexShard; import org.opensearch.threadpool.ThreadPool; @@ -167,7 +168,8 @@ public final class IndicesRequestCache implements RemovalListener> cacheEntityFunction, CacheService cacheService, ThreadPool threadPool, - ClusterService clusterService + ClusterService clusterService, + NodeEnvironment nodeEnvironment ) { this.size = INDICES_CACHE_QUERY_SIZE.get(settings); this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null; @@ -202,6 +204,7 @@ public final class IndicesRequestCache implements RemovalListener keys = new HashSet<>(); Cache cache = CacheBuilder.builder() .setMaximumWeight(numberOfEntries / 2) + .setNumberOfSegments(numberOfSegments) .removalListener(notification -> { keys.remove(notification.getKey()); evictions.incrementAndGet(); @@ -114,11 +116,13 @@ public void testCacheStats() { // check that the evicted entries were evicted in LRU order (first the odds in a batch, then the evens in a batch) // for each batch public void testCacheEvictions() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); int maximumWeight = randomIntBetween(1, numberOfEntries); AtomicLong evictions = new AtomicLong(); List evictedKeys = new ArrayList<>(); Cache cache = CacheBuilder.builder() .setMaximumWeight(maximumWeight) + .setNumberOfSegments(numberOfSegments) .removalListener(notification -> { evictions.incrementAndGet(); evictedKeys.add(notification.getKey()); @@ -173,11 +177,13 @@ public void testCacheEvictions() { // cache some entries and exceed the maximum weight, then check that the cache has the expected weight and the // expected evictions occurred public void testWeigher() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); int maximumWeight = 2 * numberOfEntries; int weight = randomIntBetween(2, 10); AtomicLong evictions = new AtomicLong(); Cache cache = CacheBuilder.builder() .setMaximumWeight(maximumWeight) + .setNumberOfSegments(numberOfSegments) .weigher((k, v) -> weight) .removalListener(notification -> evictions.incrementAndGet()) .build(); @@ -212,7 +218,8 @@ public void testWeight() { // cache some entries, randomly invalidate some of them, then check that the number of cached entries is correct public void testCount() { - Cache cache = CacheBuilder.builder().build(); + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); + Cache cache = CacheBuilder.builder().setNumberOfSegments(numberOfSegments).build(); int count = 0; for (int i = 0; i < numberOfEntries; i++) { count++; @@ -230,8 +237,9 @@ public void testCount() { // cache some entries, step the clock forward, cache some more entries, step the clock forward and then check that // the first batch of cached entries expired and were removed public void testExpirationAfterAccess() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); AtomicLong now = new AtomicLong(); - Cache cache = new Cache() { + Cache cache = new Cache(numberOfSegments) { @Override protected long now() { return now.get(); @@ -267,8 +275,9 @@ protected long now() { } public void testSimpleExpireAfterAccess() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); AtomicLong now = new AtomicLong(); - Cache cache = new Cache() { + Cache cache = new Cache(numberOfSegments) { @Override protected long now() { return now.get(); @@ -289,8 +298,9 @@ protected long now() { } public void testExpirationAfterWrite() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); AtomicLong now = new AtomicLong(); - Cache cache = new Cache() { + Cache cache = new Cache(numberOfSegments) { @Override protected long now() { return now.get(); @@ -329,8 +339,9 @@ protected long now() { } public void testComputeIfAbsentAfterExpiration() throws ExecutionException { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); AtomicLong now = new AtomicLong(); - Cache cache = new Cache() { + Cache cache = new Cache(numberOfSegments) { @Override protected long now() { return now.get(); @@ -352,8 +363,10 @@ protected long now() { } public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, InterruptedException { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); final int numberOfThreads = randomIntBetween(2, 32); final Cache cache = CacheBuilder.builder() + .setNumberOfSegments(numberOfSegments) .setExpireAfterAccess(TimeValue.timeValueNanos(1)) .build(); @@ -386,8 +399,9 @@ public void testComputeIfAbsentDeadlock() throws BrokenBarrierException, Interru // randomly promote some entries, step the clock forward, then check that the promoted entries remain and the // non-promoted entries were removed public void testPromotion() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); AtomicLong now = new AtomicLong(); - Cache cache = new Cache() { + Cache cache = new Cache(numberOfSegments) { @Override protected long now() { return now.get(); @@ -420,7 +434,8 @@ protected long now() { // randomly invalidate some cached entries, then check that a lookup for each of those and only those keys is null public void testInvalidate() { - Cache cache = CacheBuilder.builder().build(); + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); + Cache cache = CacheBuilder.builder().setNumberOfSegments(numberOfSegments).build(); for (int i = 0; i < numberOfEntries; i++) { cache.put(i, Integer.toString(i)); } @@ -443,11 +458,15 @@ public void testInvalidate() { // randomly invalidate some cached entries, then check that we receive invalidate notifications for those and only // those entries public void testNotificationOnInvalidate() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); Set notifications = new HashSet<>(); - Cache cache = CacheBuilder.builder().removalListener(notification -> { - assertEquals(RemovalReason.INVALIDATED, notification.getRemovalReason()); - notifications.add(notification.getKey()); - }).build(); + Cache cache = CacheBuilder.builder() + .setNumberOfSegments(numberOfSegments) + .removalListener(notification -> { + assertEquals(RemovalReason.INVALIDATED, notification.getRemovalReason()); + notifications.add(notification.getKey()); + }) + .build(); for (int i = 0; i < numberOfEntries; i++) { cache.put(i, Integer.toString(i)); } @@ -491,11 +510,15 @@ public void testInvalidateWithValue() { // randomly invalidate some cached entries, then check that we receive invalidate notifications for those and only // those entries public void testNotificationOnInvalidateWithValue() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); Set notifications = new HashSet<>(); - Cache cache = CacheBuilder.builder().removalListener(notification -> { - assertEquals(RemovalReason.INVALIDATED, notification.getRemovalReason()); - notifications.add(notification.getKey()); - }).build(); + Cache cache = CacheBuilder.builder() + .setNumberOfSegments(numberOfSegments) + .removalListener(notification -> { + assertEquals(RemovalReason.INVALIDATED, notification.getRemovalReason()); + notifications.add(notification.getKey()); + }) + .build(); for (int i = 0; i < numberOfEntries; i++) { cache.put(i, Integer.toString(i)); } @@ -607,8 +630,9 @@ public void testNotificationOnReplace() { } public void testComputeIfAbsentLoadsSuccessfully() { + int numberOfSegments = randomFrom(1, 2, 4, 8, 16, 64, 128, 256); Map map = new HashMap<>(); - Cache cache = CacheBuilder.builder().build(); + Cache cache = CacheBuilder.builder().setNumberOfSegments(numberOfSegments).build(); for (int i = 0; i < numberOfEntries; i++) { try { cache.computeIfAbsent(i, k -> { @@ -931,4 +955,12 @@ public void testRemoveUsingValuesIterator() { assertEquals(RemovalReason.INVALIDATED, removalNotifications.get(i).getRemovalReason()); } } + + public void testWithInvalidSegmentNumber() { + assertThrows( + "Number of segments for cache should be a power of two up-to 256", + IllegalArgumentException.class, + () -> CacheBuilder.builder().setMaximumWeight(1000).setNumberOfSegments(21).build() + ); + } } diff --git a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java index f227db6fee2d1..45a7b273eb41e 100644 --- a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java +++ b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java @@ -105,6 +105,37 @@ public void testStatsWithoutPluggableCaches() throws Exception { } } + public void testWithCacheConfigSettings() { + MockRemovalListener listener = new MockRemovalListener<>(); + int maxKeys = between(10, 50); + ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); + Settings settings = Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + 1000 + "b" // Setting some random value which shouldn't be honored. + ) + .put(FeatureFlags.PLUGGABLE_CACHE, true) + .build(); + + CacheConfig cacheConfig = new CacheConfig.Builder().setKeyType(String.class) + .setValueType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setRemovalListener(listener) + .setSettings(settings) + .setDimensionNames(dimensionNames) + .setMaxSizeInBytes(maxKeys * keyValueSize) // this should get honored + .setStatsTrackingEnabled(true) + .build(); + OpenSearchOnHeapCache onHeapCache = (OpenSearchOnHeapCache) onHeapCacheFactory.create( + cacheConfig, + CacheType.INDICES_REQUEST_CACHE, + null + ); + assertEquals(maxKeys * keyValueSize, onHeapCache.getMaximumWeight()); + } + private void assertZeroStats(ImmutableCacheStatsHolder stats) { assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), stats.getTotalStats()); } diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index 10688de3ab0ae..1a3aece74b3e2 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -73,6 +73,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentHelper; +import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.cache.request.RequestCacheStats; @@ -851,15 +852,18 @@ public void testAddingToCleanupKeyToCountMapWorksAppropriatelyWithMultipleThread assertFalse(concurrentModificationExceptionDetected.get()); } - private IndicesRequestCache getIndicesRequestCache(Settings settings) { + private IndicesRequestCache getIndicesRequestCache(Settings settings) throws IOException { IndicesService indicesService = getInstanceFromNode(IndicesService.class); - return new IndicesRequestCache( - settings, - indicesService.indicesRequestCache.cacheEntityLookup, - new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), - threadPool, - ClusterServiceUtils.createClusterService(threadPool) - ); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + return new IndicesRequestCache( + settings, + indicesService.indicesRequestCache.cacheEntityLookup, + new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), + threadPool, + ClusterServiceUtils.createClusterService(threadPool), + env + ); + } } private DirectoryReader getReader(IndexWriter writer, ShardId shardId) throws IOException { @@ -913,23 +917,26 @@ public void testClosingIndexWipesStats() throws Exception { .put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.001%") .put(FeatureFlags.PLUGGABLE_CACHE, true) .build(); - cache = new IndicesRequestCache(settings, (shardId -> { - IndexService indexService = null; - try { - indexService = indicesService.indexServiceSafe(shardId.getIndex()); - } catch (IndexNotFoundException ex) { - return Optional.empty(); - } - try { - return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id()))); - } catch (ShardNotFoundException ex) { - return Optional.empty(); - } - }), - new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), - threadPool, - ClusterServiceUtils.createClusterService(threadPool) - ); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + cache = new IndicesRequestCache(settings, (shardId -> { + IndexService indexService = null; + try { + indexService = indicesService.indexServiceSafe(shardId.getIndex()); + } catch (IndexNotFoundException ex) { + return Optional.empty(); + } + try { + return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id()))); + } catch (ShardNotFoundException ex) { + return Optional.empty(); + } + }), + new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), + threadPool, + ClusterServiceUtils.createClusterService(threadPool), + env + ); + } writer.addDocument(newDoc(0, "foo")); TermQueryBuilder termQuery = new TermQueryBuilder("id", "0"); @@ -1058,6 +1065,7 @@ public void testEviction() throws Exception { IOUtils.close(reader, secondReader, writer, dir, cache); } indexShard = createIndex("test1").getShard(0); + NodeEnvironment environment = newNodeEnvironment(); IndicesRequestCache cache = new IndicesRequestCache( // TODO: Add wiggle room to max size to allow for overhead of ICacheKey. This can be removed once API PR goes in, as it updates // the old API to account for the ICacheKey overhead. @@ -1065,7 +1073,8 @@ public void testEviction() throws Exception { (shardId -> Optional.of(new IndicesService.IndexShardCacheEntity(indexShard))), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool, - ClusterServiceUtils.createClusterService(threadPool) + ClusterServiceUtils.createClusterService(threadPool), + environment ); dir = newDirectory(); writer = new IndexWriter(dir, newIndexWriterConfig()); @@ -1085,7 +1094,7 @@ public void testEviction() throws Exception { assertEquals("baz", value3.streamInput().readString()); assertEquals(2, cache.count()); assertEquals(1, indexShard.requestCache().stats().getEvictions()); - IOUtils.close(reader, secondReader, thirdReader); + IOUtils.close(reader, secondReader, thirdReader, environment); } public void testClearAllEntityIdentity() throws Exception { From 1e49aa8625d26846ee7bea663bcc1e9296eb219a Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 8 Oct 2024 08:31:30 +0530 Subject: [PATCH 196/357] [Star tree] Add date field rounding support in star tree (#15249) --------- Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 221 ++++++++-- .../java/org/opensearch/common/Rounding.java | 18 +- .../datacube/DataCubeDateTimeUnit.java | 77 ++++ .../datacube/DateDimension.java | 110 ++++- .../compositeindex/datacube/Dimension.java | 23 ++ .../datacube/DimensionFactory.java | 18 +- .../datacube/NumericDimension.java | 17 + .../datacube/ReadDimension.java | 17 + .../datacube/startree/StarTreeField.java | 32 ++ .../startree/StarTreeIndexSettings.java | 17 +- .../builder/AbstractDocumentsFileManager.java | 8 +- .../startree/builder/BaseStarTreeBuilder.java | 90 ++++- .../builder/OffHeapStarTreeBuilder.java | 40 +- .../builder/OnHeapStarTreeBuilder.java | 35 +- .../builder/SegmentDocsFileManager.java | 12 +- .../builder/StarTreeDocsFileManager.java | 17 +- .../meta/StarTreeMetadataWriter.java | 9 +- .../utils/date/DateTimeUnitAdapter.java | 62 +++ .../utils/date/DateTimeUnitRounding.java | 29 ++ .../startree/utils/date/package-info.java | 14 + .../index/mapper/DateFieldMapper.java | 6 + .../index/mapper/StarTreeMapper.java | 65 ++- .../datacube/startree/DateDimensionTests.java | 365 +++++++++++++++++ .../datacube/startree/StarTreeTestUtils.java | 14 +- .../builder/AbstractStarTreeBuilderTests.java | 262 +++++++++++- .../index/mapper/ObjectMapperTests.java | 14 +- .../index/mapper/StarTreeMapperTests.java | 377 ++++++++++++++---- 27 files changed, 1710 insertions(+), 259 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/DataCubeDateTimeUnit.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 0d69b762ab4f2..5840884f5422a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -12,6 +12,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -22,9 +23,13 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.IndicesService; import org.opensearch.search.SearchHit; @@ -58,13 +63,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "numeric_dv_1") - .endObject() - .startObject() - .field("name", "numeric_dv_2") + .startObject("date_dimension") + .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", getDim(invalidDim, keywordDim)) .endObject() @@ -85,11 +87,58 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv_1") + .startObject("numeric") .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") .field("doc_values", true) .endObject() - .startObject("numeric_dv_2") + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static XContentBuilder createDateTestMapping(boolean duplicate) { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startObject("date_dimension") + .field("name", "timestamp") + .startArray("calendar_intervals") + .value("day") + .value("quarter-hour") + .value(duplicate ? "quarter-hour" : "half-hour") + .endArray() + .endObject() + .startArray("ordered_dimensions") + .startObject() + .field("name", "numeric_dv") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "numeric_dv") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") .field("type", "integer") .field("doc_values", true) .endObject() @@ -119,10 +168,15 @@ private static XContentBuilder createMaxDimTestMapping() { .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "dim4") + .startObject("date_dimension") + .field("name", "timestamp") + .startArray("calendar_intervals") + .value("day") + .value("month") + .value("half-hour") + .endArray() .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "dim2") .endObject() @@ -167,7 +221,7 @@ private static XContentBuilder createMaxDimTestMapping() { } } - private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + private static XContentBuilder createTestMappingWithoutStarTree() { try { return jsonBuilder().startObject() .startObject("properties") @@ -204,10 +258,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .startObject(sameStarTree ? "startree-1" : "startree-2") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "numeric_dv1") + .startObject("date_dimension") + .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", changeDim ? "numeric_new" : getDim(false, false)) .endObject() @@ -228,10 +282,6 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv1") - .field("type", "integer") - .field("doc_values", true) - .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -263,10 +313,10 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "numeric_dv2") + .startObject("date_dimension") + .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "numeric_dv") .endObject() @@ -293,10 +343,6 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv2") - .field("type", "integer") - .field("doc_values", true) - .endObject() .startObject("numeric_dv1") .field("type", "integer") .field("doc_values", true) @@ -341,8 +387,92 @@ public void testValidCompositeIndex() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); - assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY + ); + for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testValidCompositeIndexWithDates() { + prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(false)).setSettings(settings).get(); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY, + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ); + for (int i = 0; i < dateDim.getIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testValidCompositeIndexWithDuplicateDates() { + prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(true)).setSettings(settings).get(); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ); + for (int i = 0; i < dateDim.getIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); @@ -458,7 +588,7 @@ public void testUpdateIndexWithAdditionOfStarTree() { } public void testUpdateIndexWithNewerStarTree() { - prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createTestMappingWithoutStarTree(false, false, false)).get(); + prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createTestMappingWithoutStarTree()).get(); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, @@ -502,8 +632,18 @@ public void testUpdateIndexWhenMappingIsSame() { for (CompositeMappedFieldType ft : fts) { assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; - assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField()); - assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } + + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); // Assert default metrics @@ -536,6 +676,7 @@ public void testMaxDimsCompositeIndex() { MapperParsingException.class, () -> prepareCreate(TEST_INDEX).setSettings(settings) .setMapping(createMaxDimTestMapping()) + // Date dimension is considered as one dimension regardless of number of actual calendar intervals .setSettings( Settings.builder() .put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2) @@ -569,6 +710,24 @@ public void testMaxMetricsCompositeIndex() { ); } + public void testMaxCalendarIntervalsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings( + Settings.builder() + .put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + ) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", + ex.getMessage() + ); + } + public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java index 6f5f1e4328758..12a399635046e 100644 --- a/server/src/main/java/org/opensearch/common/Rounding.java +++ b/server/src/main/java/org/opensearch/common/Rounding.java @@ -95,7 +95,7 @@ public enum DateTimeUnit { WEEK_OF_WEEKYEAR((byte) 1, "week", IsoFields.WEEK_OF_WEEK_BASED_YEAR, true, TimeUnit.DAYS.toMillis(7)) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundWeekOfWeekYear(utcMillis); } @@ -107,7 +107,7 @@ long extraLocalOffsetLookup() { YEAR_OF_CENTURY((byte) 2, "year", ChronoField.YEAR_OF_ERA, false, 12) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundYear(utcMillis); } @@ -118,7 +118,7 @@ long extraLocalOffsetLookup() { QUARTER_OF_YEAR((byte) 3, "quarter", IsoFields.QUARTER_OF_YEAR, false, 3) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundQuarterOfYear(utcMillis); } @@ -129,7 +129,7 @@ long extraLocalOffsetLookup() { MONTH_OF_YEAR((byte) 4, "month", ChronoField.MONTH_OF_YEAR, false, 1) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundMonthOfYear(utcMillis); } @@ -138,7 +138,7 @@ long extraLocalOffsetLookup() { } }, DAY_OF_MONTH((byte) 5, "day", ChronoField.DAY_OF_MONTH, true, ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis()) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, this.ratio); } @@ -147,7 +147,7 @@ long extraLocalOffsetLookup() { } }, HOUR_OF_DAY((byte) 6, "hour", ChronoField.HOUR_OF_DAY, true, ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis()) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -162,7 +162,7 @@ long extraLocalOffsetLookup() { true, ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis() ) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -177,7 +177,7 @@ long extraLocalOffsetLookup() { true, ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis() ) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -210,7 +210,7 @@ public long extraLocalOffsetLookup() { * @param utcMillis the milliseconds since the epoch * @return the rounded down milliseconds since the epoch */ - abstract long roundFloor(long utcMillis); + public abstract long roundFloor(long utcMillis); /** * When looking up {@link LocalTimeOffset} go this many milliseconds diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DataCubeDateTimeUnit.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DataCubeDateTimeUnit.java new file mode 100644 index 0000000000000..56b704103dcae --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DataCubeDateTimeUnit.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.unmodifiableMap; + +/** + * Enum representing the extended date time units supported for star tree index as part of index mapping. + * The enum values are: + *

      + *
    • HALF_HOUR_OF_DAY: Represents half hour of day rounding
      • + *
    • QUARTER_HOUR_OF_DAY: Represents quarter hour of day rounding
      • + *
    + *

    + * The enum also provides a static map of date field units to their corresponding ExtendedDateTimeUnit instances. + * + * @see org.opensearch.common.Rounding.DateTimeUnit for more information on the dateTimeUnit enum and rounding logic. + * + * @opensearch.experimental + */ +public enum DataCubeDateTimeUnit implements DateTimeUnitRounding { + HALF_HOUR_OF_DAY("half-hour") { + @Override + public long roundFloor(long utcMillis) { + return utcMillis - (utcMillis % TimeUnit.MINUTES.toMillis(30)); + } + }, + QUARTER_HOUR_OF_DAY("quarter-hour") { + @Override + public long roundFloor(long utcMillis) { + return utcMillis - (utcMillis % TimeUnit.MINUTES.toMillis(15)); + } + }; + + public static final Map DATE_FIELD_UNITS; + static { + Map dateFieldUnits = new HashMap<>(); + dateFieldUnits.put("30m", DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("half-hour", DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("15m", DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY); + dateFieldUnits.put("quarter-hour", DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY); + DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits); + } + + private final String shortName; + + DataCubeDateTimeUnit(String shortName) { + this.shortName = shortName; + } + + /** + * This rounds down the supplied milliseconds since the epoch down to the next unit. In order to retain performance this method + * should be as fast as possible and not try to convert dates to java-time objects if possible + * + * @param utcMillis the milliseconds since the epoch + * @return the rounded down milliseconds since the epoch + */ + @Override + public abstract long roundFloor(long utcMillis); + + @Override + public String shortName() { + return shortName; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index 074016db2aed7..ee6d5b4680c73 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -10,12 +10,21 @@ import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.time.DateUtils; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.DateFieldMapper; import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; +import java.util.stream.Collectors; /** * Date dimension class @@ -24,27 +33,78 @@ */ @ExperimentalApi public class DateDimension implements Dimension { - private final List calendarIntervals; + private final List calendarIntervals; public static final String CALENDAR_INTERVALS = "calendar_intervals"; public static final String DATE = "date"; private final String field; + private final List sortedCalendarIntervals; + private final DateFieldMapper.Resolution resolution; - public DateDimension(String field, List calendarIntervals) { + public DateDimension(String field, List calendarIntervals, DateFieldMapper.Resolution resolution) { this.field = field; this.calendarIntervals = calendarIntervals; + // Sort from the lowest unit to the highest unit + this.sortedCalendarIntervals = getSortedDateTimeUnits(calendarIntervals); + if (resolution == null) { + this.resolution = DateFieldMapper.Resolution.MILLISECONDS; + } else { + this.resolution = resolution; + } } - public List getIntervals() { + public List getIntervals() { return calendarIntervals; } + public List getSortedCalendarIntervals() { + return sortedCalendarIntervals; + } + + /** + * Sets the dimension values in sorted order in the provided array starting from the given index. + * + * @param val The value to be set + * @param dimSetter Consumer which sets the dimensions + */ + @Override + public void setDimensionValues(final Long val, final Consumer dimSetter) { + for (DateTimeUnitRounding dateTimeUnit : sortedCalendarIntervals) { + if (val == null) { + dimSetter.accept(null); + } else { + Long roundedValue = dateTimeUnit.roundFloor(storedDurationSinceEpoch(val)); + dimSetter.accept(roundedValue); + } + } + } + + /** + * Converts nanoseconds to milliseconds based on the resolution of the field + */ + private long storedDurationSinceEpoch(long nanoSecondsSinceEpoch) { + if (resolution.equals(DateFieldMapper.Resolution.NANOSECONDS)) return DateUtils.toMilliSeconds(nanoSecondsSinceEpoch); + return nanoSecondsSinceEpoch; + } + + /** + * Returns the list of fields that represent the dimension + */ + @Override + public List getSubDimensionNames() { + List fields = new ArrayList<>(calendarIntervals.size()); + for (DateTimeUnitRounding interval : sortedCalendarIntervals) { + fields.add(field + "_" + interval.shortName()); + } + return fields; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); + builder.startObject("date_dimension"); builder.field(CompositeDataCubeFieldType.NAME, this.getField()); builder.field(CompositeDataCubeFieldType.TYPE, DATE); builder.startArray(CALENDAR_INTERVALS); - for (Rounding.DateTimeUnit interval : calendarIntervals) { + for (DateTimeUnitRounding interval : calendarIntervals) { builder.value(interval.shortName()); } builder.endArray(); @@ -69,4 +129,44 @@ public int hashCode() { public String getField() { return field; } + + @Override + public int getNumSubDimensions() { + return calendarIntervals.size(); + } + + /** + * DateTimeUnit Comparator which tracks dateTimeUnits in sorted order from second unit to year unit + */ + public static class DateTimeUnitComparator implements Comparator { + public static final Map ORDERED_DATE_TIME_UNIT = new HashMap<>(); + + static { + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.SECOND_OF_MINUTE.shortName(), 1); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), 2); + ORDERED_DATE_TIME_UNIT.put(DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY.shortName(), 3); + ORDERED_DATE_TIME_UNIT.put(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY.shortName(), 4); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.HOUR_OF_DAY.shortName(), 5); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.DAY_OF_MONTH.shortName(), 6); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR.shortName(), 7); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MONTH_OF_YEAR.shortName(), 8); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.QUARTER_OF_YEAR.shortName(), 9); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.YEAR_OF_CENTURY.shortName(), 10); + } + + @Override + public int compare(DateTimeUnitRounding unit1, DateTimeUnitRounding unit2) { + return Integer.compare( + ORDERED_DATE_TIME_UNIT.getOrDefault(unit1.shortName(), Integer.MAX_VALUE), + ORDERED_DATE_TIME_UNIT.getOrDefault(unit2.shortName(), Integer.MAX_VALUE) + ); + } + } + + /** + * Returns a sorted list of dateTimeUnits based on the DateTimeUnitComparator + */ + public static List getSortedDateTimeUnits(List dateTimeUnits) { + return dateTimeUnits.stream().sorted(new DateTimeUnitComparator()).collect(Collectors.toList()); + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index 0151a474579be..cfa8d3a2a8164 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -11,6 +11,9 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; +import java.util.List; +import java.util.function.Consumer; + /** * Base interface for data-cube dimensions * @@ -18,5 +21,25 @@ */ @ExperimentalApi public interface Dimension extends ToXContent { + String getField(); + + /** + * Returns the number of dimension values that gets added to star tree document + * as part of this dimension + */ + int getNumSubDimensions(); + + /** + * Sets the dimension values with the consumer + * + * @param value The value to be set + * @param dimSetter Consumer which sets the dimensions + */ + void setDimensionValues(final Long value, final Consumer dimSetter); + + /** + * Returns the list of dimension fields that represent the dimension + */ + List getSubDimensionNames(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 3c418c68fe8ad..7e72a3f0d9de6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -8,16 +8,19 @@ package org.opensearch.index.compositeindex.datacube; -import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.Mapper; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS; @@ -70,13 +73,13 @@ private static DateDimension parseAndCreateDateDimension( Map dimensionMap, Mapper.TypeParser.ParserContext c ) { - List calendarIntervals = new ArrayList<>(); + Set calendarIntervals; List intervalStrings = XContentMapValues.extractRawValues(CALENDAR_INTERVALS, dimensionMap) .stream() .map(Object::toString) .collect(Collectors.toList()); if (intervalStrings == null || intervalStrings.isEmpty()) { - calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings()); + calendarIntervals = new LinkedHashSet<>(StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings())); } else { if (intervalStrings.size() > StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings())) { throw new IllegalArgumentException( @@ -88,12 +91,17 @@ private static DateDimension parseAndCreateDateDimension( ) ); } + calendarIntervals = new LinkedHashSet<>(); for (String interval : intervalStrings) { calendarIntervals.add(StarTreeIndexSettings.getTimeUnit(interval)); } - calendarIntervals = new ArrayList<>(calendarIntervals); } dimensionMap.remove(CALENDAR_INTERVALS); - return new DateDimension(name, calendarIntervals); + DateFieldMapper.Resolution resolution = null; + if (c != null && c.mapperService() != null && c.mapperService().fieldType(name) != null) { + resolution = ((DateFieldMapper.DateFieldType) c.mapperService().fieldType(name)).resolution(); + } + + return new DateDimension(name, new ArrayList<>(calendarIntervals), resolution); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index 9c25ef5b25503..acc14f5f05c68 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -13,7 +13,9 @@ import org.opensearch.index.mapper.CompositeDataCubeFieldType; import java.io.IOException; +import java.util.List; import java.util.Objects; +import java.util.function.Consumer; /** * Composite index numeric dimension class @@ -33,6 +35,21 @@ public String getField() { return field; } + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public void setDimensionValues(final Long val, final Consumer dimSetter) { + dimSetter.accept(val); + } + + @Override + public List getSubDimensionNames() { + return List.of(field); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 4264ec87d2c74..be3667f10b6da 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -12,7 +12,9 @@ import org.opensearch.index.mapper.CompositeDataCubeFieldType; import java.io.IOException; +import java.util.List; import java.util.Objects; +import java.util.function.Consumer; /** * Represents a dimension for reconstructing StarTreeField from file formats during searches and merges. @@ -31,6 +33,21 @@ public String getField() { return field; } + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public void setDimensionValues(final Long val, final Consumer dimSetter) { + dimSetter.accept(val); + } + + @Override + public List getSubDimensionNames() { + return List.of(field); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java index 922ddcbea4fe2..833bf63c04a18 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java @@ -11,10 +11,13 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -29,12 +32,24 @@ public class StarTreeField implements ToXContent { private final List dimensionsOrder; private final List metrics; private final StarTreeFieldConfiguration starTreeConfig; + private final List dimensionNames; + private final List metricNames; public StarTreeField(String name, List dimensions, List metrics, StarTreeFieldConfiguration starTreeConfig) { this.name = name; this.dimensionsOrder = dimensions; this.metrics = metrics; this.starTreeConfig = starTreeConfig; + dimensionNames = new ArrayList<>(); + for (Dimension dimension : dimensions) { + dimensionNames.addAll(dimension.getSubDimensionNames()); + } + metricNames = new ArrayList<>(); + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getMetrics()) { + metricNames.add(metric.getField() + "_" + metricStat.name()); + } + } } public String getName() { @@ -45,6 +60,14 @@ public List getDimensionsOrder() { return dimensionsOrder; } + public List getDimensionNames() { + return dimensionNames; + } + + public List getMetricNames() { + return metricNames; + } + public List getMetrics() { return metrics; } @@ -57,13 +80,22 @@ public StarTreeFieldConfiguration getStarTreeConfig() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("name", name); + DateDimension dateDim = null; if (dimensionsOrder != null && !dimensionsOrder.isEmpty()) { builder.startArray("ordered_dimensions"); for (Dimension dimension : dimensionsOrder) { + // Handle dateDimension for later + if (dimension instanceof DateDimension) { + dateDim = (DateDimension) dimension; + continue; + } dimension.toXContent(builder, params); } builder.endArray(); } + if (dateDim != null) { + dateDim.toXContent(builder, params); + } if (metrics != null && !metrics.isEmpty()) { builder.startArray("metrics"); for (Metric metric : metrics) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index e665831b83d93..72be8ac44cbbc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -10,7 +10,10 @@ import org.opensearch.common.Rounding; import org.opensearch.common.settings.Setting; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.Arrays; @@ -97,9 +100,9 @@ public class StarTreeIndexSettings { /** * Default intervals for date dimension as part of star tree fields */ - public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( "index.composite_index.star_tree.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), DataCubeDateTimeUnit.HALF_HOUR_OF_DAY.shortName()), StarTreeIndexSettings::getTimeUnit, Setting.Property.IndexScope, Setting.Property.Final @@ -116,11 +119,13 @@ public class StarTreeIndexSettings { Setting.Property.Final ); - public static Rounding.DateTimeUnit getTimeUnit(String expression) { - if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { - throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); + public static DateTimeUnitRounding getTimeUnit(String expression) { + if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + return new DateTimeUnitAdapter(DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression)); + } else if (DataCubeDateTimeUnit.DATE_FIELD_UNITS.containsKey(expression)) { + return DataCubeDateTimeUnit.DATE_FIELD_UNITS.get(expression); } - return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); } public static final Setting IS_COMPOSITE_INDEX_SETTING = Setting.boolSetting( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java index 327fd26c00608..3f4e302e0a0f2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java @@ -43,17 +43,20 @@ public abstract class AbstractDocumentsFileManager implements Closeable { protected final TrackingDirectoryWrapper tmpDirectory; protected final SegmentWriteState state; protected int docSizeInBytes = -1; + protected final int numDimensions; public AbstractDocumentsFileManager( SegmentWriteState state, StarTreeField starTreeField, - List metricAggregatorInfos + List metricAggregatorInfos, + int numDimensions ) { this.starTreeField = starTreeField; this.tmpDirectory = new TrackingDirectoryWrapper(state.directory); this.metricAggregatorInfos = metricAggregatorInfos; this.state = state; numMetrics = metricAggregatorInfos.size(); + this.numDimensions = numDimensions; } private void setDocSizeInBytes(int numBytes) { @@ -124,8 +127,7 @@ protected int writeMetrics(StarTreeDocument starTreeDocument, IndexOutput output * @throws IOException IOException in case of I/O errors */ protected StarTreeDocument readStarTreeDocument(RandomAccessInput input, long offset, boolean isAggregatedDoc) throws IOException { - int dimSize = starTreeField.getDimensionsOrder().size(); - Long[] dimensions = new Long[dimSize]; + Long[] dimensions = new Long[numDimensions]; long initialOffset = offset; offset = readDimensions(dimensions, input, offset); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 5e5814528fcd2..2d4938eeb45b3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -18,6 +18,7 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedNumericDocValuesWriterWrapper; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Counter; import org.apache.lucene.util.NumericUtils; @@ -52,6 +53,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; @@ -82,7 +84,7 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected int totalSegmentDocs; protected int numStarTreeNodes; protected final int maxLeafDocuments; - + List dimensionsSplitOrder = new ArrayList<>(); protected final InMemoryTreeNode rootNode = getNewNode(); protected final StarTreeField starTreeField; @@ -112,9 +114,12 @@ protected BaseStarTreeBuilder( this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); - - List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); - this.numDimensions = dimensionsSplitOrder.size(); + int numDims = 0; + for (Dimension dim : starTreeField.getDimensionsOrder()) { + numDims += dim.getNumSubDimensions(); + dimensionsSplitOrder.add(dim); + } + this.numDimensions = numDims; this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = writeState.segmentInfo.maxDoc(); @@ -122,9 +127,12 @@ protected BaseStarTreeBuilder( Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); - for (int i = 0; i < numDimensions; i++) { + for (int i = 0; i < dimensionsSplitOrder.size(); i++) { if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { - this.skipStarNodeCreationForDimensions.add(i); + // add the dimension indices + for (int dimIndex = 0; dimIndex < dimensionsSplitOrder.get(i).getNumSubDimensions(); dimIndex++) { + this.skipStarNodeCreationForDimensions.add(i + dimIndex); + } } } @@ -224,7 +232,7 @@ public void build( List metricReaders = getMetricReaders(writeState, fieldProducerMap); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - for (int i = 0; i < numDimensions; i++) { + for (int i = 0; i < dimensionReaders.length; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); if (dimensionFieldInfo == null) { @@ -313,19 +321,20 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A throws IOException { List dimensionWriters = new ArrayList<>(); List metricWriters = new ArrayList<>(); - FieldInfo[] dimensionFieldInfoList = new FieldInfo[starTreeField.getDimensionsOrder().size()]; + FieldInfo[] dimensionFieldInfoList = new FieldInfo[numDimensions]; FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; - for (int i = 0; i < dimensionFieldInfoList.length; i++) { - final FieldInfo fi = getFieldInfo( - fullyQualifiedFieldNameForStarTreeDimensionsDocValues( - starTreeField.getName(), - starTreeField.getDimensionsOrder().get(i).getField() - ), - DocValuesType.SORTED_NUMERIC, - fieldNumberAcrossStarTrees.getAndIncrement() - ); - dimensionFieldInfoList[i] = fi; - dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + int dimIndex = 0; + for (Dimension dim : dimensionsSplitOrder) { + for (String name : dim.getSubDimensionNames()) { + final FieldInfo fi = getFieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), name), + DocValuesType.SORTED_NUMERIC, + fieldNumberAcrossStarTrees.getAndIncrement() + ); + dimensionFieldInfoList[dimIndex] = fi; + dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + dimIndex++; + } } for (int i = 0; i < metricAggregatorInfos.size(); i++) { @@ -371,7 +380,7 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A } } - addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, starTreeField.getDimensionsOrder().size()); + addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, numDimensions); addStarTreeDocValueFields(docValuesConsumer, metricWriters, metricFieldInfoList, metricAggregatorInfos.size()); } @@ -422,6 +431,36 @@ protected StarTreeDocument getStarTreeDocument( return new StarTreeDocument(dims, metrics); } + /** + * Sets dimensions / metric readers nnd numSegmentDocs + */ + protected void setReadersAndNumSegmentDocs( + SequentialDocValuesIterator[] dimensionReaders, + List metricReaders, + AtomicInteger numSegmentDocs, + StarTreeValues starTreeValues + ) { + List dimensionNames = starTreeValues.getStarTreeField().getDimensionNames(); + for (int i = 0; i < numDimensions; i++) { + dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimensionNames.get(i))); + } + // get doc id set iterators for metrics + for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeValues.getStarTreeField().getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); + } + } + + numSegmentDocs.set( + Integer.parseInt(starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS))) + ); + } + /** * Adds a document to the star-tree. * @@ -500,7 +539,8 @@ protected StarTreeDocument getSegmentStarTreeDocument( */ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIterator[] dimensionReaders) throws IOException { Long[] dimensions = new Long[numDimensions]; - for (int i = 0; i < numDimensions; i++) { + AtomicInteger dimIndex = new AtomicInteger(0); + for (int i = 0; i < dimensionReaders.length; i++) { if (dimensionReaders[i] != null) { try { dimensionReaders[i].nextEntry(currentDocId); @@ -511,11 +551,16 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte logger.error("unable to read the dimension values from the segment", e); throw new IllegalStateException("unable to read the dimension values from the segment", e); } - dimensions[i] = dimensionReaders[i].value(currentDocId); + dimensionsSplitOrder.get(i).setDimensionValues(dimensionReaders[i].value(currentDocId), value -> { + dimensions[dimIndex.getAndIncrement()] = value; + }); } else { throw new IllegalStateException("dimension readers are empty"); } } + if (dimIndex.get() != numDimensions) { + throw new IllegalStateException("Values are not set for all dimensions"); + } return dimensions; } @@ -685,6 +730,7 @@ private SequentialDocValuesIterator getIteratorForNumericField( * @throws IOException throws an exception if we are unable to add the doc */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { + appendStarTreeDocument(starTreeDocument); numStarTreeDocs++; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 11cd9e0c94113..9c39555396397 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -12,13 +12,9 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.index.compositeindex.datacube.Dimension; -import org.opensearch.index.compositeindex.datacube.Metric; -import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -36,9 +32,6 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; - /** * Off-heap implementation of the star tree builder. * @@ -68,9 +61,9 @@ protected OffHeapStarTreeBuilder( MapperService mapperService ) throws IOException { super(metaOut, dataOut, starTreeField, state, mapperService); - segmentDocumentFileManager = new SegmentDocsFileManager(state, starTreeField, metricAggregatorInfos); + segmentDocumentFileManager = new SegmentDocsFileManager(state, starTreeField, metricAggregatorInfos, numDimensions); try { - starTreeDocumentFileManager = new StarTreeDocsFileManager(state, starTreeField, metricAggregatorInfos); + starTreeDocumentFileManager = new StarTreeDocsFileManager(state, starTreeField, metricAggregatorInfos, numDimensions); } catch (IOException e) { IOUtils.closeWhileHandlingException(segmentDocumentFileManager); throw e; @@ -147,31 +140,12 @@ Iterator mergeStarTrees(List starTreeValuesSub int[] docIds; try { for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[starTreeValues.getStarTreeField() - .getDimensionsOrder() - .size()]; - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); - } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - // get doc id set iterators for metrics - for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getBaseMetrics()) { - String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( - starTreeValues.getStarTreeField().getName(), - metric.getField(), - metricStat.getTypeName() - ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); - } - } + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); segmentDocumentFileManager.writeStarTreeDocument(starTreeDocument, true); numDocs++; @@ -317,7 +291,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start int docId = 1; private boolean hasSameDimensions(StarTreeDocument document1, StarTreeDocument document2) { - for (int i = dimensionId + 1; i < starTreeField.getDimensionsOrder().size(); i++) { + for (int i = dimensionId + 1; i < numDimensions; i++) { if (!Objects.equals(document1.dimensions[i], document2.dimensions[i])) { return false; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 13c6d03c4dc3d..07142fc5c8be7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -9,12 +9,8 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.compositeindex.datacube.Dimension; -import org.opensearch.index.compositeindex.datacube.Metric; -import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -29,9 +25,6 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; - /** * On heap single tree builder * @@ -133,33 +126,13 @@ Iterator mergeStarTrees(List starTreeValuesSub StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeValuesSubs) throws IOException { List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); - } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - // get doc id set iterators for metrics - for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getBaseMetrics()) { - String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( - starTreeValues.getStarTreeField().getName(), - metric.getField(), - metricStat.getTypeName() - ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); - - } - } - + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); currentDocId++; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java index fe94df57d9535..363e02b52d5e3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java @@ -40,9 +40,13 @@ public class SegmentDocsFileManager extends AbstractDocumentsFileManager impleme private RandomAccessInput segmentRandomInput; final IndexOutput segmentDocsFileOutput; - public SegmentDocsFileManager(SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos) - throws IOException { - super(state, starTreeField, metricAggregatorInfos); + public SegmentDocsFileManager( + SegmentWriteState state, + StarTreeField starTreeField, + List metricAggregatorInfos, + int numDimensions + ) throws IOException { + super(state, starTreeField, metricAggregatorInfos, numDimensions); try { segmentDocsFileOutput = tmpDirectory.createTempOutput(SEGMENT_DOC_FILE_NAME, state.segmentSuffix, state.context); } catch (IOException e) { @@ -78,7 +82,7 @@ public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) @Override public Long[] readDimensions(int docId) throws IOException { maybeInitializeSegmentInput(); - Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; + Long[] dims = new Long[numDimensions]; readDimensions(dims, segmentRandomInput, (long) docId * docSizeInBytes); return dims; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 15ed153249243..7e920b912731d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -60,18 +60,23 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; - public StarTreeDocsFileManager(SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos) - throws IOException { - this(state, starTreeField, metricAggregatorInfos, DEFAULT_FILE_COUNT_MERGE_THRESHOLD); + public StarTreeDocsFileManager( + SegmentWriteState state, + StarTreeField starTreeField, + List metricAggregatorInfos, + int numDimensions + ) throws IOException { + this(state, starTreeField, metricAggregatorInfos, DEFAULT_FILE_COUNT_MERGE_THRESHOLD, numDimensions); } public StarTreeDocsFileManager( SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos, - int fileCountThreshold + int fileCountThreshold, + int numDimensions ) throws IOException { - super(state, starTreeField, metricAggregatorInfos); + super(state, starTreeField, metricAggregatorInfos, numDimensions); fileToEndDocIdMap = new LinkedHashMap<>(); try { starTreeDocsFileOutput = createStarTreeDocumentsFileOutput(); @@ -126,7 +131,7 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { @Override public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); - Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; + Long[] dims = new Long[numDimensions]; readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId)); return dims; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 1c04350e25047..42e6f3c59866a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.mapper.CompositeMappedFieldType; @@ -128,13 +127,11 @@ private static void writeMeta( metaOut.writeVInt(numNodes); // number of dimensions - // TODO: Revisit the number of dimensions for timestamps (as we will split timestamp into min, hour, etc.) - metaOut.writeVInt(starTreeField.getDimensionsOrder().size()); + metaOut.writeVInt(starTreeField.getDimensionNames().size()); // dimensions - // TODO: Add sub-dimensions for timestamps (as we will split timestamp into min, hour, etc.) - for (Dimension dimension : starTreeField.getDimensionsOrder()) { - metaOut.writeString(dimension.getField()); + for (String dim : starTreeField.getDimensionNames()) { + metaOut.writeString(dim); } // number of metrics diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java new file mode 100644 index 0000000000000..b18ac33a8f657 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.date; + +import org.opensearch.common.Rounding; + +import java.util.Objects; + +/** + * Adapter class to convert {@link Rounding.DateTimeUnit} to {@link DateTimeUnitRounding} + * + * @opensearch.experimental + */ +public class DateTimeUnitAdapter implements DateTimeUnitRounding { + private final Rounding.DateTimeUnit dateTimeUnit; + + public DateTimeUnitAdapter(Rounding.DateTimeUnit dateTimeUnit) { + this.dateTimeUnit = dateTimeUnit; + } + + @Override + public String shortName() { + return dateTimeUnit.shortName(); + } + + @Override + public long roundFloor(long utcMillis) { + return dateTimeUnit.roundFloor(utcMillis); + } + + /** + * Checks if this DateTimeUnitRounding is equal to another object. + * Two DateTimeUnitRounding instances are considered equal if they have the same short name. + * + * @param obj The object to compare with + * @return true if the objects are equal, false otherwise + */ + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof DateTimeUnitRounding)) return false; + DateTimeUnitRounding other = (DateTimeUnitRounding) obj; + return Objects.equals(shortName(), other.shortName()); + } + + /** + * Returns a hash code value for the object. + * This method is implemented to be consistent with equals. + * + * @return a hash code value for this object + */ + @Override + public int hashCode() { + return Objects.hash(shortName()); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java new file mode 100644 index 0000000000000..93212a2424e46 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.date; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Interface for rounding time units in starTree + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface DateTimeUnitRounding { + /** + * Returns the short name of the time unit + */ + String shortName(); + + /** + * rounds down the given utcMillis to the nearest unit of time + */ + long roundFloor(long utcMillis); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java new file mode 100644 index 0000000000000..bb285c894e3e0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Classes and utilities for working with dateTimeUnits + * + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils.date; diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index bc50f507010c4..4698440f08036 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -54,6 +54,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.LocaleUtils; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -76,6 +77,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -350,6 +352,10 @@ public DateFieldMapper build(BuilderContext context) { return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } + @Override + public Optional getSupportedDataCubeDimensionType() { + return Optional.of(DimensionType.DATE); + } } public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> { diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index 52dab17e0b0bb..40f05a8b76755 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Query; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.DimensionFactory; import org.opensearch.index.compositeindex.datacube.Metric; @@ -44,8 +45,8 @@ public class StarTreeMapper extends ParametrizedFieldMapper { public static final String CONFIG = "config"; public static final String MAX_LEAF_DOCS = "max_leaf_docs"; public static final String SKIP_STAR_NODE_IN_DIMS = "skip_star_node_creation_for_dimensions"; - public static final String BUILD_MODE = "build_mode"; public static final String ORDERED_DIMENSIONS = "ordered_dimensions"; + public static final String DATE_DIMENSION = "date_dimension"; public static final String METRICS = "metrics"; public static final String STATS = "stats"; @@ -84,6 +85,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { StarTreeFieldConfiguration.StarTreeBuildMode buildMode = StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP; List dimensions = buildDimensions(name, paramMap, context); + paramMap.remove(DATE_DIMENSION); paramMap.remove(ORDERED_DIMENSIONS); List metrics = buildMetrics(name, paramMap, context); paramMap.remove(METRICS); @@ -119,16 +121,21 @@ public static class Builder extends ParametrizedFieldMapper.Builder { */ @SuppressWarnings("unchecked") private List buildDimensions(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + List dimensions = new LinkedList<>(); + DateDimension dateDim = buildDateDimension(fieldName, map, context); + if (dateDim != null) { + dimensions.add(dateDim); + } Object dims = XContentMapValues.extractValue("ordered_dimensions", map); if (dims == null) { throw new IllegalArgumentException( String.format(Locale.ROOT, "ordered_dimensions is required for star tree field [%s]", fieldName) ); } - List dimensions = new LinkedList<>(); + if (dims instanceof List) { - List dimList = (List) dims; - if (dimList.size() > context.getSettings() + List orderedDimensionsList = (List) dims; + if (orderedDimensionsList.size() + dimensions.size() > context.getSettings() .getAsInt( StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_DEFAULT @@ -146,13 +153,13 @@ private List buildDimensions(String fieldName, Map ma ) ); } - if (dimList.size() < 2) { + if (dimensions.size() + orderedDimensionsList.size() < 2) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Atleast two dimensions are required to build star tree index field [%s]", fieldName) ); } Set dimensionFieldNames = new HashSet<>(); - for (Object dim : dimList) { + for (Object dim : orderedDimensionsList) { Dimension dimension = getDimension(fieldName, dim, context); if (dimensionFieldNames.add(dimension.getField()) == false) { throw new IllegalArgumentException( @@ -174,6 +181,52 @@ private List buildDimensions(String fieldName, Map ma return dimensions; } + private DateDimension buildDateDimension(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + Object dims = XContentMapValues.extractValue("date_dimension", map); + if (dims == null) { + return null; + } + return getDateDimension(fieldName, dims, context); + } + + /** + * Get dimension based on mapping + */ + @SuppressWarnings("unchecked") + private DateDimension getDateDimension(String fieldName, Object dimensionMapping, Mapper.TypeParser.ParserContext context) { + DateDimension dimension; + Map dimensionMap = (Map) dimensionMapping; + String name = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.NAME, dimensionMap); + dimensionMap.remove(CompositeDataCubeFieldType.NAME); + if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) { + String type = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.TYPE, dimensionMap); + dimensionMap.remove(CompositeDataCubeFieldType.TYPE); + if (type == null || type.equals(DateDimension.DATE) == false) { + throw new MapperParsingException( + String.format(Locale.ROOT, "unable to parse date dimension for star tree field [%s]", fieldName) + ); + } + return (DateDimension) DimensionFactory.parseAndCreateDimension(name, type, dimensionMap, context); + } else { + Optional dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); + if (dimBuilder.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown date dimension field [%s]", name)); + } + if (dimBuilder.get() instanceof DateFieldMapper.Builder == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "date_dimension [%s] should be of type date for star tree field [%s]", name, fieldName) + ); + } + dimension = (DateDimension) DimensionFactory.parseAndCreateDimension(name, dimBuilder.get(), dimensionMap, context); + } + DocumentMapperParser.checkNoRemainingFields( + dimensionMap, + context.indexVersionCreated(), + "Star tree mapping definition has unsupported parameters: " + ); + return dimension; + } + /** * Get dimension based on mapping */ diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java new file mode 100644 index 0000000000000..0aa563ed0cafd --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java @@ -0,0 +1,365 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.Rounding; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.datacube.DateDimension.DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT; + +public class DateDimensionTests extends OpenSearchTestCase { + public void testDateDimension() { + String field = "timestamp"; + List intervals = Arrays.asList( + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY) + ); + DateDimension dateDimension = new DateDimension(field, intervals, DateFieldMapper.Resolution.MILLISECONDS); + + assertEquals(field, dateDimension.getField()); + assertEquals(intervals, dateDimension.getIntervals()); + for (int i = 0; i < intervals.size(); i++) { + assertEquals(intervals.get(i).shortName(), dateDimension.getSortedCalendarIntervals().get(i).shortName()); + } + } + + public void testSetDimensionValuesWithMultipleIntervalsYear() { + List intervals = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) + ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] dims = new Long[4]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); + + assertEquals(4, dimIndex.get()); + assertEquals(1609459200000L, (long) dims[0]); // Hour rounded + assertEquals(1609459200000L, (long) dims[1]); // Day rounded + assertEquals(1609459200000L, (long) dims[2]); // Month rounded + assertEquals(1609459200000L, (long) dims[3]); // Year rounded + assertEquals(4, dateDimension.getNumSubDimensions()); + } + + public void testSetDimensionValuesForHalfAndQuarterHour() { + List intervals = Arrays.asList( + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY + ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] dims = new Long[2]; + long testValue = 1724230620123L; // August 21, 2024 8:57:00.123 UTC + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex.get()); + assertEquals(1724229900000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:45:00 UTC + assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + Long[] dims1 = new Long[2]; + testValue = 1724229899234L; // Wed, 21 Aug 2024 08:44:59 GMT + + AtomicInteger dimIndex1 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims1[dimIndex1.getAndIncrement()] = value; }); + + assertEquals(1724229000000L, (long) dims1[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims1[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + Long[] dims2 = new Long[2]; + testValue = 1724229000123L; // Wed, 21 Aug 2024 08:30:00 GMT + + AtomicInteger dimIndex2 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims2[dimIndex2.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex2.get()); + assertEquals(1724229000000L, (long) dims2[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims2[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + Long[] dims3 = new Long[2]; + testValue = 1724228940000L; // Wed, 21 Aug 2024 08:29:00 GMT + + AtomicInteger dimIndex3 = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims3[dimIndex3.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex3.get()); + assertEquals(1724228100000L, (long) dims3[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:15:00 UTC + assertEquals(1724227200000L, (long) dims3[1]); // Half hour rounded - Wed, 21 Aug 2024 08:00:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + } + + public void testRoundingAndSortingAllDateTimeUnitsNanos() { + List allUnits = getAllTimeUnits(); + + DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.NANOSECONDS); + + // Test sorting + List sortedUnits = dateDimension.getSortedCalendarIntervals(); + assertEquals(allUnits.size(), sortedUnits.size()); + for (int i = 0; i < sortedUnits.size() - 1; i++) { + assertTrue( + "Units should be sorted in ascending order", + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i).shortName()) + .compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1).shortName())) <= 0 + ); + } + + // Test rounding + long testValueNanos = 1655293505382719622L; // 2022-06-15T11:45:05.382719622Z + Long[] dims = new Long[allUnits.size()]; + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValueNanos, value -> { dims[dimIndex.getAndIncrement()] = value; }); + + // Expected rounded values (in nanoseconds) + long secondRounded = 1655293505000L; // 2022-06-15T11:45:05Z + long minuteRounded = 1655293500000L; // 2022-06-15T11:45:00Z + long quarterHourRounded = 1655293500000L; // 2022-06-15T11:45:00Z + long halfHourRounded = 1655292600000L; // 2022-06-15T11:30:00Z + long hourRounded = 1655290800000L; // 2022-06-15T11:00:00Z + long dayRounded = 1655251200000L; // 2022-06-15T00:00:00Z + long weekRounded = 1655078400000L; // 2022-06-13T00:00:00Z (Monday) + long monthRounded = 1654041600000L; // 2022-06-01T00:00:00Z + long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00Z + long yearRounded = 1640995200000L; // 2022-01-01T00:00:00Z + + assertTimeUnits( + sortedUnits, + dims, + secondRounded, + minuteRounded, + hourRounded, + dayRounded, + weekRounded, + monthRounded, + quarterRounded, + yearRounded, + halfHourRounded, + quarterHourRounded + ); + } + + public void testRoundingAndSortingAllDateTimeUnitsMillis() { + List allUnits = getAllTimeUnits(); + + DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.MILLISECONDS); + + // Test sorting + List sortedUnits = dateDimension.getSortedCalendarIntervals(); + assertEquals(allUnits.size(), sortedUnits.size()); + for (int i = 0; i < sortedUnits.size() - 1; i++) { + assertTrue( + "Units should be sorted in ascending order", + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i).shortName()) + .compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1).shortName())) <= 0 + ); + } + + // Test rounding + long testValueNanos = 1724114825234L; // 2024-08-20T00:47:05.234Z + Long[] dims = new Long[allUnits.size()]; + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValueNanos, value -> { dims[dimIndex.getAndIncrement()] = value; }); + + // Expected rounded values (in millis) + long secondRounded = 1724114825000L; // 2024-08-20T00:47:05.000Z + long minuteRounded = 1724114820000L; // 2024-08-20T00:47:00.000Z + long quarterHourRounded = 1724114700000L; // 2024-08-20T00:45:00.000Z + long halfHourRounded = 1724113800000L; // 2024-08-20T00:30:00.000Z + long hourRounded = 1724112000000L; // 2024-08-20T00:00:00.000Z + long dayRounded = 1724112000000L; // 2024-08-20T00:00:00.000Z + long weekRounded = 1724025600000L; // 2024-08-15T00:00:00.000Z (Monday) + long monthRounded = 1722470400000L; // 2024-08-01T00:00:00.000Z + long quarterRounded = 1719792000000L; // 2024-07-01T00:00:00.000Z + long yearRounded = 1704067200000L; // 2024-01-01T00:00:00.000Z + + assertTimeUnits( + sortedUnits, + dims, + secondRounded, + minuteRounded, + hourRounded, + dayRounded, + weekRounded, + monthRounded, + quarterRounded, + yearRounded, + halfHourRounded, + quarterHourRounded + ); + } + + private static List getAllTimeUnits() { + return Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.QUARTER_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.SECOND_OF_MINUTE), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY + ); + } + + private static void assertTimeUnits( + List sortedUnits, + Long[] dims, + long secondRounded, + long minuteRounded, + long hourRounded, + long dayRounded, + long weekRounded, + long monthRounded, + long quarterRounded, + long yearRounded, + long halfHourRounded, + long quarterHourRounded + ) { + for (int i = 0; i < sortedUnits.size(); i++) { + DateTimeUnitRounding unit = sortedUnits.get(i); + String unitName = unit.shortName(); + switch (unitName) { + case "second": + assertEquals(secondRounded, (long) dims[i]); + break; + case "minute": + assertEquals(minuteRounded, (long) dims[i]); + break; + case "hour": + assertEquals(hourRounded, (long) dims[i]); + break; + case "day": + assertEquals(dayRounded, (long) dims[i]); + break; + case "week": + assertEquals(weekRounded, (long) dims[i]); + break; + case "month": + assertEquals(monthRounded, (long) dims[i]); + break; + case "quarter": + assertEquals(quarterRounded, (long) dims[i]); + break; + case "year": + assertEquals(yearRounded, (long) dims[i]); + break; + case "half-hour": + assertEquals(halfHourRounded, (long) dims[i]); + break; + case "quarter-hour": + assertEquals(quarterHourRounded, (long) dims[i]); + break; + default: + fail("Unexpected DateTimeUnit: " + unit); + } + } + } + + public void testGetSubDimensionNames() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ), + DateFieldMapper.Resolution.MILLISECONDS + ); + + List fields = dateDimension.getSubDimensionNames(); + + assertEquals(2, fields.size()); + assertEquals("timestamp_hour", fields.get(0)); + assertEquals("timestamp_day", fields.get(1)); + } + + public void testGetExtendedTimeUnitFieldsNames() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, DataCubeDateTimeUnit.QUARTER_HOUR_OF_DAY), + DateFieldMapper.Resolution.MILLISECONDS + ); + + List fields = dateDimension.getSubDimensionNames(); + + assertEquals(2, fields.size()); + assertEquals("timestamp_quarter-hour", fields.get(0)); + assertEquals("timestamp_half-hour", fields.get(1)); + } + + public void testSetDimensionValues() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) + ), + DateFieldMapper.Resolution.MILLISECONDS + ); + Long[] dims = new Long[2]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + + AtomicInteger dimIndex = new AtomicInteger(0); + dateDimension.setDimensionValues(testValue, value -> { dims[dimIndex.getAndIncrement()] = value; }); + + assertEquals(2, dimIndex.get()); + assertEquals(1609459200000L, (long) dims[0]); // Hour rounded + assertEquals(1609459200000L, (long) dims[1]); // Year rounded + } + + public void testDateTimeUnitComparator() { + Comparator comparator = new DateDimension.DateTimeUnitComparator(); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.SECOND_OF_MINUTE), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR) + ) < 0 + ); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ) < 0 + ); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) + ) > 0 + ); + assertEquals( + 0, + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR) + ) + ); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 7cae1cd25ee93..dc8b3320f3de2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -50,11 +50,19 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + int numDimensions = 0; + for (Dimension dimension : dimensionsSplitOrder) { + numDimensions += dimension.getNumSubDimensions(); + } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; + int dimIndex = 0; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); + Dimension dimension = dimensionsSplitOrder.get(i); + for (String name : dimension.getSubDimensionNames()) { + dimensionReaders[dimIndex] = new SequentialDocValuesIterator(starTreeValues.getDimensionValuesIterator(name)); + dimIndex++; + } } List metricReaders = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 95fd1579c2f63..0d4e4c37d6924 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -33,12 +33,15 @@ import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; +import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; import org.opensearch.index.codec.composite.composite99.Composite99Codec; import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; import org.opensearch.index.compositeindex.CompositeIndexConstants; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -53,9 +56,12 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldValueConverter; import org.opensearch.index.mapper.Mapper; @@ -84,6 +90,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; @@ -1864,7 +1871,7 @@ private List getStarTreeDimensionNames(List dimensionsOrder) List dimensionNames = new ArrayList<>(); for (Dimension dimension : dimensionsOrder) { - dimensionNames.add(dimension.getField()); + dimensionNames.addAll(dimension.getSubDimensionNames()); } return dimensionNames; @@ -3842,7 +3849,7 @@ public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOE } public static long getLongFromDouble(double value) { - return Double.doubleToLongBits(value); + return NumericUtils.doubleToSortableLong(value); } public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOException { @@ -4199,9 +4206,11 @@ public void testMergeFlow() throws IOException { ... [999, 999, 999, 999] | [19980.0] */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); - assertEquals(2L, starTreeDocument.metrics[1]); + assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[1]); + assertEquals(2L, starTreeDocument.metrics[2]); } builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); @@ -4213,7 +4222,7 @@ public void testMergeFlow() throws IOException { docValuesConsumer.close(); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", + compositeField.getName(), STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, @@ -4237,6 +4246,251 @@ public void testMergeFlow() throws IOException { ); } + public void testFlushFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithDateDimension(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(6, writeState.segmentInfo.getId()), mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [1655287920000, 1655287200000, 1655287200000, 4] | [40.0, 1] + [1655287980000, 1655287200000, 1655287200000, 3] | [30.0, 1] + [1655288040000, 1655287200000, 1655287200000, 1] | [10.0, 1] + [1655288040000, 1655287200000, 1655287200000, 5] | [50.0, 1] + [1655288100000, 1655287200000, 1655287200000, 0] | [0.0, 1] + [null, null, null, 2] | [20.0, 1] + */ + int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + starTreeDocumentIterator = starTreeDocumentsList.iterator(); + while (starTreeDocumentIterator.hasNext()) { + count++; + StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); + assertEquals(starTreeDocument.dimensions[3] * 1 * 10.0, starTreeDocument.metrics[1]); + assertEquals(1L, starTreeDocument.metrics[0]); + } + assertEquals(6, count); + builder.build(starTreeDocumentsList.iterator(), new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); + } + + public void testMergeFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L); + List docsWithField = List.of(0, 1, 2, 3, 4, 6); + List dimList2 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); + List dimList7 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField7 = List.of(0, 1, 2, 3, 4, 6); + + List dimList5 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List docsWithField5 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList1 = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField1 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField4 = List.of(0, 1, 3, 4); + List dimList8 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField8 = List.of(0, 1, 3, 4); + + List dimList6 = List.of(5L, 6L, 7L, 8L); + List docsWithField6 = List.of(0, 1, 2, 3); + List metricsList21 = List.of( + getLongFromDouble(50.0), + getLongFromDouble(60.0), + getLongFromDouble(70.0), + getLongFromDouble(80.0), + getLongFromDouble(90.0) + ); + List metricsWithField21 = List.of(0, 1, 2, 3, 4); + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeFieldWithDateDimension(); + StarTreeValues starTreeValues = getStarTreeValuesWithDates( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(dimList7, docsWithField7), + getSortedNumericMock(dimList5, docsWithField5), + getSortedNumericMock(metricsList, metricsWithField), + getSortedNumericMock(metricsList1, metricsWithField1), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValuesWithDates( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(dimList8, docsWithField8), + getSortedNumericMock(dimList6, docsWithField6), + getSortedNumericMock(metricsList2, metricsWithField2), + getSortedNumericMock(metricsList21, metricsWithField21), + compositeField, + "4" + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [1655287972000, 1655287972000, 1655287972000, 3] | [30.0, 3] + [1655288032000, 1655288032000, 1655288032000, 2] | [20.0, 2] + [1655288032000, 1655288032000, 1655288032000, 8] | [80.0, 8] + [1655288092000, 1655288092000, 1655288092000, 1] | [10.0, 1] + [1655288092000, 1655288092000, 1655288092000, 4] | [40.0, 4] + [1655288092000, 1655288092000, 1655288092000, 6] | [60.0, 6] + [1655288152000, 1655288152000, 1655288152000, 0] | [0.0, 0] + [1655288152000, 1655288152000, 1655288152000, 5] | [50.0, 5] + [null, null, null, 5] | [50.0, 5] + [null, null, null, 7] | [70.0, 7] + */ + int count = 0; + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[1], 0); + assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[0]); + } + assertEquals(10, count); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 10, + builder.numStarTreeDocs, + compositeField.getStarTreeConfig().maxLeafDocs(), + Set.of(), + getBuildMode(), + 0, + 231 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + private StarTreeValues getStarTreeValuesWithDates( + SortedNumericDocValues dimList, + SortedNumericDocValues dimList2, + SortedNumericDocValues dimList4, + SortedNumericDocValues dimList3, + SortedNumericDocValues metricsList, + SortedNumericDocValues metricsList1, + StarTreeField sf, + String number + ) { + Map> dimDocIdSetIterators = Map.of( + "field1_minute", + () -> new SortedNumericStarTreeValuesIterator(dimList), + "field1_half-hour", + () -> new SortedNumericStarTreeValuesIterator(dimList4), + "field1_hour", + () -> new SortedNumericStarTreeValuesIterator(dimList2), + "field3", + () -> new SortedNumericStarTreeValuesIterator(dimList3) + ); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList) + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(1).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList1) + ); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number), null); + } + + private StarTreeField getStarTreeFieldWithDateDimension() { + List intervals = new ArrayList<>(); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + intervals.add(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.VALUE_COUNT, MetricStat.SUM)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(10, new HashSet<>(), getBuildMode()); + StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + return sf; + } + private void validateStarTree( InMemoryTreeNode root, int totalDimensions, diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index b415e1e657f7f..c35cf3fc1e591 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -506,10 +506,10 @@ public void testCompositeFields() throws Exception { .startObject("startree") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "node") + .startObject("date_dimension") + .field("name", "@timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "status") .endObject() @@ -526,8 +526,8 @@ public void testCompositeFields() throws Exception { .endObject() .endObject() .startObject("properties") - .startObject("node") - .field("type", "integer") + .startObject("@timestamp") + .field("type", "date") .endObject() .startObject("status") .field("type", "integer") @@ -565,9 +565,9 @@ public void testCompositeFields() throws Exception { StarTreeMapper starTreeMapper = (StarTreeMapper) mapper; assertEquals("star_tree", starTreeMapper.fieldType().typeName()); // Check that field in properties was parsed correctly as well - mapper = documentMapper.root().getMapper("node"); + mapper = documentMapper.root().getMapper("@timestamp"); assertNotNull(mapper); - assertEquals("integer", mapper.typeName()); + assertEquals("date", mapper.typeName()); FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index daa9fda7a5a3b..aac460bd5e332 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -18,6 +18,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.compositeindex.CompositeIndexValidator; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; @@ -27,6 +28,8 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.junit.After; import org.junit.Before; @@ -42,6 +45,7 @@ import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING; import static org.opensearch.index.compositeindex.CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING; import static org.hamcrest.Matchers.containsString; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; /** * Tests for {@link StarTreeMapper}. @@ -74,7 +78,16 @@ public void testValidStarTree() throws IOException { for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; assertEquals(2, starTreeFieldType.getDimensions().size()); - assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(2, starTreeFieldType.getMetrics().size()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -85,7 +98,7 @@ public void testValidStarTree() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -130,7 +143,16 @@ public void testMetricsWithJustSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.MONTH_OF_YEAR + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(new DateTimeUnitAdapter(expectedTimeUnits.get(i)), dateDim.getIntervals().get(i)); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -140,7 +162,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -151,7 +173,16 @@ public void testMetricsWithCountAndSum() throws IOException { Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.MONTH_OF_YEAR + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(new DateTimeUnitAdapter(expectedTimeUnits.get(i)), dateDim.getIntervals().get(i)); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); @@ -166,18 +197,47 @@ public void testMetricsWithCountAndSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } } + public void testValidStarTreeWithNoDateDim() throws IOException { + MapperService mapperService = createMapperService(getMinMappingWithDateDims(false, true, true)); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("status", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof NumericDimension); + assertEquals("metric_field", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof NumericDimension); + assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals(new HashSet<>(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + public void testValidStarTreeDefaults() throws IOException { MapperService mapperService = createMapperService(getMinMapping()); Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); for (CompositeMappedFieldType type : compositeFieldTypes) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; - assertEquals("node", starTreeFieldType.getDimensions().get(0).getField()); + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedDimensionFields = Arrays.asList("@timestamp_minute", "@timestamp_half-hour"); + assertEquals(expectedDimensionFields, dateDim.getSubDimensionNames()); + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals(3, starTreeFieldType.getMetrics().size()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); @@ -196,6 +256,56 @@ public void testValidStarTreeDefaults() throws IOException { } } + public void testValidStarTreeDateDims() throws IOException { + MapperService mapperService = createMapperService(getMinMappingWithDateDims(false, false, false)); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedDimensionFields = Arrays.asList("@timestamp_half-hour", "@timestamp_week", "@timestamp_month"); + assertEquals(expectedDimensionFields, dateDim.getSubDimensionNames()); + List expectedTimeUnits = Arrays.asList( + DataCubeDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) + ); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + + public void testInValidStarTreeMinDims() throws IOException { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMappingWithDateDims(false, true, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: Atleast two dimensions are required to build star tree index field [startree]", + ex.getMessage() + ); + } + + public void testInvalidStarTreeDateDims() throws IOException { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMappingWithDateDims(true, false, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [3] calendar intervals are allowed in dimension [@timestamp]", + ex.getMessage() + ); + } + public void testInvalidDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, @@ -215,7 +325,7 @@ public void testInvalidMetric() { public void testNoMetrics() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, true, false, false)) + () -> createMapperService(getMinMapping(false, true, false, false, false)) ); assertThat( ex.getMessage(), @@ -226,7 +336,7 @@ public void testNoMetrics() { public void testInvalidParam() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, false, false, true, false)) + () -> createMapperService(getInvalidMapping(false, false, false, false, true, false, false)) ); assertEquals( "Failed to parse mapping [_doc]: Star tree mapping definition has unsupported parameters: [invalid : {invalid=invalid}]", @@ -237,7 +347,7 @@ public void testInvalidParam() { public void testNoDims() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(true, false, false, false)) + () -> createMapperService(getMinMapping(true, false, false, false, false)) ); assertThat( ex.getMessage(), @@ -245,18 +355,26 @@ public void testNoDims() { ); } + public void testMissingDateDims() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(false, false, false, false, true)) + ); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown date dimension field [@timestamp]")); + } + public void testMissingDims() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, false, true, false)) + () -> createMapperService(getMinMapping(false, false, true, false, false)) ); - assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [node]")); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [status]")); } public void testMissingMetrics() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, false, false, true)) + () -> createMapperService(getMinMapping(false, false, false, true, false)) ); assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown metric field [metric_field]")); } @@ -275,7 +393,7 @@ public void testInvalidMetricType() { public void testInvalidMetricTypeWithDocCount() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, false, false, false, true)) + () -> createMapperService(getInvalidMapping(false, false, false, false, false, true, false)) ); assertEquals("Failed to parse mapping [_doc]: Invalid metric stat: _doc_count", ex.getMessage()); } @@ -286,7 +404,18 @@ public void testInvalidDimType() { () -> createMapperService(getInvalidMapping(false, false, true, false)) ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [node] as part of star tree field [startree]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [status] as part of star tree field [startree]", + ex.getMessage() + ); + } + + public void testInvalidDateDimType() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, false, true, false, false, false, true)) + ); + assertEquals( + "Failed to parse mapping [_doc]: date_dimension [@timestamp] should be of type date for star tree field [startree]", ex.getMessage() ); } @@ -356,17 +485,17 @@ public void testMetric() { } public void testDimensions() { - List d1CalendarIntervals = new ArrayList<>(); - d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - DateDimension d1 = new DateDimension("name", d1CalendarIntervals); - DateDimension d2 = new DateDimension("name", d1CalendarIntervals); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); + DateDimension d2 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertEquals(d1, d2); - d2 = new DateDimension("name1", d1CalendarIntervals); + d2 = new DateDimension("name1", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); - List d2CalendarIntervals = new ArrayList<>(); - d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - d2 = new DateDimension("name", d2CalendarIntervals); + List d2CalendarIntervals = new ArrayList<>(); + d2CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + d2CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + d2 = new DateDimension("name", d2CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); NumericDimension n1 = new NumericDimension("name"); NumericDimension n2 = new NumericDimension("name"); @@ -387,9 +516,9 @@ public void testStarTreeField() { List m1 = new ArrayList<>(); m1.add(MetricStat.MAX); Metric metric1 = new Metric("name", m1); - List d1CalendarIntervals = new ArrayList<>(); - d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); NumericDimension n1 = new NumericDimension("numeric"); NumericDimension n2 = new NumericDimension("name1"); @@ -483,7 +612,7 @@ public void testValidations() throws IOException { ) ); assertEquals( - "Aggregations not supported for the dimension field [node] with field type [integer] as part of star tree field", + "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field", ex.getMessage() ); @@ -506,14 +635,18 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); - b.field("name", "node"); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -530,8 +663,8 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -551,10 +684,10 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() - .field("name", "node") + .startObject("date_dimension") + .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "numeric_dv") .endObject() @@ -574,8 +707,8 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .endObject() .endObject() .startObject("properties") - .startObject("node") - .field("type", "integer") + .startObject("timestamp") + .field("type", "date") .endObject() .startObject("numeric_dv") .field("type", "integer") @@ -602,14 +735,18 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); - b.field("name", "node"); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -626,8 +763,8 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -648,14 +785,18 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); - b.field("name", "node"); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -673,8 +814,8 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -687,21 +828,95 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met } private XContentBuilder getMinMapping() throws IOException { - return getMinMapping(false, false, false, false); + return getMinMapping(false, false, false, false, false); } - private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetrics, boolean missingDim, boolean missingMetric) + private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExceeded, boolean dateDimsAbsent, boolean additionalDim) throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); - if (!isEmptyDims) { - b.startArray("ordered_dimensions"); + if (!dateDimsAbsent) { + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value(getRandom().nextBoolean() ? "week" : "1w"); + if (calendarIntervalsExceeded) { + b.value(getRandom().nextBoolean() ? "day" : "1d"); + b.value(getRandom().nextBoolean() ? "second" : "1s"); + b.value(getRandom().nextBoolean() ? "hour" : "1h"); + b.value(getRandom().nextBoolean() ? "minute" : "1m"); + b.value(getRandom().nextBoolean() ? "year" : "1y"); + b.value(getRandom().nextBoolean() ? "quarter-hour" : "15m"); + } + b.value(getRandom().nextBoolean() ? "month" : "1M"); + b.value(getRandom().nextBoolean() ? "half-hour" : "30m"); + b.endArray(); + b.endObject(); + } + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + if (additionalDim) { b.startObject(); - b.field("name", "node"); + b.field("name", "metric_field"); b.endObject(); + } + b.endArray(); + + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + + b.endObject(); + + b.endObject(); + b.endObject(); + b.startObject("properties"); + + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + + b.startObject("metric_field"); + b.field("type", "integer"); + b.endObject(); + + b.endObject(); + }); + } + + private XContentBuilder getMinMapping( + boolean isEmptyDims, + boolean isEmptyMetrics, + boolean missingDim, + boolean missingMetric, + boolean missingDateDim + ) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + if (!isEmptyDims) { + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", "status"); b.endObject(); @@ -721,14 +936,16 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric b.endObject(); b.endObject(); b.startObject("properties"); + if (!missingDateDim) { + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + } if (!missingDim) { - b.startObject("node"); + b.startObject("status"); b.field("type", "integer"); b.endObject(); } - b.startObject("status"); - b.field("type", "integer"); - b.endObject(); if (!missingMetric) { b.startObject("metric_field"); b.field("type", "integer"); @@ -812,14 +1029,14 @@ private XContentBuilder getInvalidMapping( boolean invalidDimType, boolean invalidMetricType, boolean invalidParam, - boolean invalidDocCountMetricType + boolean invalidDocCountMetricType, + boolean invalidDate ) throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); b.field("type", "star_tree"); b.startObject("config"); - b.startArray("skip_star_node_creation_for_dimensions"); { if (invalidSkipDims) { @@ -828,6 +1045,9 @@ private XContentBuilder getInvalidMapping( b.value("status"); } b.endArray(); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.endObject(); if (invalidParam) { b.startObject("invalid"); b.field("invalid", "invalid"); @@ -836,12 +1056,9 @@ private XContentBuilder getInvalidMapping( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "node"); + b.field("name", "status"); b.endObject(); } - b.startObject(); - b.field("name", "status"); - b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -861,16 +1078,20 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - if (!invalidDimType) { - b.field("type", "integer"); + b.startObject("@timestamp"); + if (!invalidDate) { + b.field("type", "date"); } else { b.field("type", "keyword"); } b.endObject(); b.startObject("status"); - b.field("type", "integer"); + if (!invalidDimType) { + b.field("type", "integer"); + } else { + b.field("type", "keyword"); + } b.endObject(); b.startObject("metric_field"); if (invalidMetricType) { @@ -903,15 +1124,15 @@ private XContentBuilder getInvalidMappingWithDv( b.value("status"); } b.endArray(); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.endObject(); b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "node"); + b.field("name", "status"); b.endObject(); } - b.startObject(); - b.field("name", "status"); - b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -925,12 +1146,12 @@ private XContentBuilder getInvalidMappingWithDv( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); + b.startObject("@timestamp"); if (!invalidDimType) { - b.field("type", "integer"); + b.field("type", "date"); b.field("doc_values", "true"); } else { - b.field("type", "integer"); + b.field("type", "date"); b.field("doc_values", "false"); } b.endObject(); @@ -953,7 +1174,7 @@ private XContentBuilder getInvalidMappingWithDv( private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType) throws IOException { - return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false, false); + return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false, false, false); } protected boolean supportsOrIgnoresBoost() { From 3696c296220acf4bf0ecb1bc10cb4a71675d4939 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 8 Oct 2024 10:03:53 +0530 Subject: [PATCH 197/357] [Star tree] Refactoring builder tests (#16036) * Refactoring builder tests Signed-off-by: Bharathwaj G * adding date tests Signed-off-by: Bharathwaj G --------- Signed-off-by: Bharathwaj G --- .../builder/AbstractStarTreeBuilderTests.java | 4740 ----------------- .../startree/builder/BuilderTestsUtils.java | 526 ++ .../builder/OffHeapStarTreeBuilderTests.java | 36 - .../builder/OnHeapStarTreeBuilderTests.java | 37 - .../builder/StarTreeBuildMetricTests.java | 954 ++++ .../StarTreeBuilderFlushFlowTests.java | 419 ++ .../StarTreeBuilderMergeFlowTests.java | 1921 +++++++ .../StarTreeBuilderSortAndAggregateTests.java | 695 +++ .../builder/StarTreeBuilderTestCase.java | 368 ++ 9 files changed, 4883 insertions(+), 4813 deletions(-) delete mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java delete mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java delete mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderSortAndAggregateTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java deleted file mode 100644 index 0d4e4c37d6924..0000000000000 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ /dev/null @@ -1,4740 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.builder; - -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.EmptyDocValuesProducer; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.sandbox.document.HalfFloatPoint; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.Version; -import org.opensearch.common.Rounding; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; -import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; -import org.opensearch.index.compositeindex.CompositeIndexConstants; -import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; -import org.opensearch.index.compositeindex.datacube.DateDimension; -import org.opensearch.index.compositeindex.datacube.Dimension; -import org.opensearch.index.compositeindex.datacube.Metric; -import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.NumericDimension; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; -import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; -import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; -import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; -import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; -import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; -import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; -import org.opensearch.index.mapper.ContentPath; -import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.index.mapper.DocumentMapper; -import org.opensearch.index.mapper.FieldValueConverter; -import org.opensearch.index.mapper.Mapper; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.MappingLookup; -import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; - -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; -import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; -import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; -import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public abstract class AbstractStarTreeBuilderTests extends OpenSearchTestCase { - protected MapperService mapperService; - protected List dimensionsOrder; - protected List fields = List.of(); - protected List metrics; - protected Directory directory; - protected FieldInfo[] fieldsInfo; - protected StarTreeField compositeField; - protected Map fieldProducerMap; - protected SegmentWriteState writeState; - protected BaseStarTreeBuilder builder; - protected IndexOutput dataOut; - protected IndexOutput metaOut; - protected DocValuesConsumer docValuesConsumer; - protected String dataFileName; - protected String metaFileName; - - @Before - public void setup() throws IOException { - fields = List.of("field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", "field9", "field10"); - - dimensionsOrder = List.of( - new NumericDimension("field1"), - new NumericDimension("field3"), - new NumericDimension("field5"), - new NumericDimension("field8") - ); - metrics = List.of( - new Metric("field2", List.of(MetricStat.SUM)), - new Metric("field4", List.of(MetricStat.SUM)), - new Metric("field6", List.of(MetricStat.VALUE_COUNT)), - new Metric("field9", List.of(MetricStat.MIN)), - new Metric("field10", List.of(MetricStat.MAX)), - new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)) - ); - - DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - - compositeField = new StarTreeField( - "test", - dimensionsOrder, - metrics, - new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) - ); - directory = newFSDirectory(createTempDir()); - - fieldsInfo = new FieldInfo[fields.size()]; - fieldProducerMap = new HashMap<>(); - for (int i = 0; i < fieldsInfo.length; i++) { - fieldsInfo[i] = new FieldInfo( - fields.get(i), - i, - false, - false, - true, - IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - fieldProducerMap.put(fields.get(i), docValuesProducer); - } - writeState = getWriteState(5, UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8)); - - dataFileName = IndexFileNames.segmentFileName( - writeState.segmentInfo.name, - writeState.segmentSuffix, - Composite99DocValuesFormat.DATA_EXTENSION - ); - dataOut = writeState.directory.createOutput(dataFileName, writeState.context); - - metaFileName = IndexFileNames.segmentFileName( - writeState.segmentInfo.name, - writeState.segmentSuffix, - Composite99DocValuesFormat.META_EXTENSION - ); - metaOut = writeState.directory.createOutput(metaFileName, writeState.context); - - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - docValuesConsumer = mock(DocValuesConsumer.class); - } - - private SegmentReadState getReadState(int numDocs, List dimensionFields, List metrics) { - - int numMetrics = 0; - for (Metric metric : metrics) { - numMetrics += metric.getBaseMetrics().size(); - } - - FieldInfo[] fields = new FieldInfo[dimensionFields.size() + numMetrics]; - - int i = 0; - for (String dimension : dimensionFields) { - fields[i] = new FieldInfo( - fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeField.getName(), dimension), - i, - false, - false, - true, - IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - i++; - } - - for (Metric metric : metrics) { - for (MetricStat metricStat : metric.getBaseMetrics()) { - fields[i] = new FieldInfo( - fullyQualifiedFieldNameForStarTreeMetricsDocValues( - compositeField.getName(), - metric.getField(), - metricStat.getTypeName() - ), - i, - false, - false, - true, - IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - i++; - } - } - - SegmentInfo segmentInfo = new SegmentInfo( - directory, - Version.LATEST, - Version.LUCENE_9_11_0, - "test_segment", - numDocs, - false, - false, - new Lucene912Codec(), - new HashMap<>(), - writeState.segmentInfo.getId(), - new HashMap<>(), - null - ); - return new SegmentReadState(segmentInfo.dir, segmentInfo, new FieldInfos(fields), writeState.context); - } - - private SegmentWriteState getWriteState(int numDocs, byte[] id) { - FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - SegmentInfo segmentInfo = new SegmentInfo( - directory, - Version.LATEST, - Version.LUCENE_9_12_0, - "test_segment", - numDocs, - false, - false, - new Lucene912Codec(), - new HashMap<>(), - id, - new HashMap<>(), - null - ); - return new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); - } - - public abstract BaseStarTreeBuilder getStarTreeBuilder( - IndexOutput metaOut, - IndexOutput dataOut, - StarTreeField starTreeField, - SegmentWriteState segmentWriteState, - MapperService mapperService - ) throws IOException; - - public void test_sortAndAggregateStarTreeDocuments() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0, 10L } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, 10L } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0, 10L } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, null } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, null } - ); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = (Long) starTreeDocuments[i].metrics[5]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 11L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 21L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - int numOfAggregatedDocuments = 0; - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - - numOfAggregatedDocuments++; - } - - assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - } - - SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeDocuments) { - SequentialDocValuesIterator[] sequentialDocValuesIterators = - new SequentialDocValuesIterator[starTreeDocuments[0].dimensions.length]; - for (int j = 0; j < starTreeDocuments[0].dimensions.length; j++) { - List dimList = new ArrayList<>(); - List docsWithField = new ArrayList<>(); - - for (int i = 0; i < starTreeDocuments.length; i++) { - if (starTreeDocuments[i].dimensions[j] != null) { - dimList.add(starTreeDocuments[i].dimensions[j]); - docsWithField.add(i); - } - } - sequentialDocValuesIterators[j] = new SequentialDocValuesIterator( - new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) - ); - } - return sequentialDocValuesIterators; - } - - List getMetricIterators(StarTreeDocument[] starTreeDocuments) { - List sequentialDocValuesIterators = new ArrayList<>(); - for (int j = 0; j < starTreeDocuments[0].metrics.length; j++) { - List metricslist = new ArrayList<>(); - List docsWithField = new ArrayList<>(); - - for (int i = 0; i < starTreeDocuments.length; i++) { - if (starTreeDocuments[i].metrics[j] != null) { - metricslist.add((long) starTreeDocuments[i].metrics[j]); - docsWithField.add(i); - } - } - sequentialDocValuesIterators.add( - new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) - ); - } - return sequentialDocValuesIterators; - } - - public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 }); - starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 }); - starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 }); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 }); - starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 11.0, null, randomDouble(), 8.0, 13.0 }); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 18.0, 3L, 6.0, 24.0, 3L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - } - - public void test_sortAndAggregateStarTreeDocuments_nullMetricField() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - // Setting second metric iterator as empty sorted numeric , indicating a metric field is null - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, null } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, null } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, null } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, null } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 0.0, 2L, 8.0, 20.0, 11L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 0.0, 3L, 6.0, 24.0, 3L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = starTreeDocuments[i].metrics[5] != null ? (Long) starTreeDocuments[i].metrics[5] : null; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - } - - public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionField() throws IOException { - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - // Setting second metric iterator as empty sorted numeric , indicating a metric field is null - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, null, 3L, 4L }, - new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, null } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { null, 4L, 2L, 1L }, - new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, null } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, 4L, 2L, 1L }, - new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, null } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, null, 3L, 4L }, - new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { -1L, 4L, 2L, 1L }, - new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, null } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 11.0, 0.0, 1L, 8.0, 13.0, 1L }), - new StarTreeDocument(new Long[] { 2L, null, 3L, 4L }, new Object[] { 21.0, 0.0, 2L, 8.0, 20.0, 11L }), - new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 24.0, 0.0, 2L, 6.0, 24.0, 2L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = starTreeDocuments[i].metrics[5] != null ? (long) starTreeDocuments[i].metrics[5] : null; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - - assertFalse(expectedStarTreeDocumentIterator.hasNext()); - - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); - } - - public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics() throws IOException { - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - // Setting second metric iterator as empty sorted numeric , indicating a metric field is null - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, null, null, null } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, null, null, null } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, null, null, null } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, null, null, null } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, null, null, null } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { 0.0, 0.0, 0L, null, null, 5L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - Long metric1 = starTreeDocuments[i].metrics[0] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]) - : null; - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - Long metric3 = starTreeDocuments[i].metrics[2] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]) - : null; - Long metric4 = starTreeDocuments[i].metrics[3] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]) - : null; - Long metric5 = starTreeDocuments[i].metrics[4] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]) - : null; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); - } - - public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetrics() throws IOException { - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - double sumValue = randomDouble(); - double minValue = randomDouble(); - double maxValue = randomDouble(); - - // Setting second metric iterator as empty sorted numeric , indicating a metric field is null - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, randomDouble(), null, maxValue } - ); - starTreeDocuments[1] = new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { null, null, null, null, null }); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { null, null, null, minValue, null } - ); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { null, null, null, null, null }); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { sumValue, null, randomDouble(), null, null } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { sumValue, 0.0, 2L, minValue, maxValue, 5L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - Long metric1 = starTreeDocuments[i].metrics[0] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]) - : null; - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - Long metric3 = starTreeDocuments[i].metrics[2] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]) - : null; - Long metric4 = starTreeDocuments[i].metrics[3] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]) - : null; - Long metric5 = starTreeDocuments[i].metrics[4] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]) - : null; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); - } - - public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - // Setting second metric iterator as empty sorted numeric , indicating a metric field is null - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, 10L } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, 10L } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, 10L } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { null, null, null, null }, - new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, 10L } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { 56.0, 0.0, 5L, 6.0, 24.0, 50L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - Long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - Long metric2 = starTreeDocuments[i].metrics[1] != null - ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) - : null; - Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - Long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - Long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = (Long) starTreeDocuments[i].metrics[5]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - } - } - - public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, - new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, - new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, - new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, - new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, - new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0 } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - int numOfAggregatedDocuments = 0; - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); - - numOfAggregatedDocuments++; - } - - assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - - } - - public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { Double.MAX_VALUE, 10.0, randomDouble(), 8.0, 20.0, 100L } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, 100L } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 14.0, Double.MIN_VALUE, randomDouble(), 6.0, 24.0, 100L } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, 100L } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, 100L } - ); - - List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { Double.MAX_VALUE + 9, 14.0, 2L, 8.0, 20.0, 200L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, Double.MIN_VALUE + 22, 3L, 6.0, 24.0, 300L }) - ); - Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = (Long) starTreeDocuments[i].metrics[5]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - int numOfAggregatedDocuments = 0; - while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - - numOfAggregatedDocuments++; - } - - assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); - - } - - public void test_build_halfFloatMetrics() throws IOException { - - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder( - "field10", - NumberFieldMapper.NumberType.HALF_FLOAT, - false, - true - ).build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new HalfFloatPoint[] { - new HalfFloatPoint("hf1", 12), - new HalfFloatPoint("hf6", 10), - new HalfFloatPoint("field6", 10), - new HalfFloatPoint("field9", 8), - new HalfFloatPoint("field10", 20) } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new HalfFloatPoint[] { - new HalfFloatPoint("hf2", 10), - new HalfFloatPoint("hf7", 6), - new HalfFloatPoint("field6", 10), - new HalfFloatPoint("field9", 12), - new HalfFloatPoint("field10", 10) } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new HalfFloatPoint[] { - new HalfFloatPoint("hf3", 14), - new HalfFloatPoint("hf8", 12), - new HalfFloatPoint("field6", 10), - new HalfFloatPoint("field9", 6), - new HalfFloatPoint("field10", 24) } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new HalfFloatPoint[] { - new HalfFloatPoint("hf4", 9), - new HalfFloatPoint("hf9", 4), - new HalfFloatPoint("field6", 10), - new HalfFloatPoint("field9", 9), - new HalfFloatPoint("field10", 12) } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new HalfFloatPoint[] { - new HalfFloatPoint("hf5", 11), - new HalfFloatPoint("hf10", 16), - new HalfFloatPoint("field6", 10), - new HalfFloatPoint("field9", 8), - new HalfFloatPoint("field10", 13) } - ); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = HalfFloatPoint.halfFloatToSortableShort( - ((HalfFloatPoint) starTreeDocuments[i].metrics[0]).numericValue().floatValue() - ); - long metric2 = HalfFloatPoint.halfFloatToSortableShort( - ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() - ); - long metric3 = HalfFloatPoint.halfFloatToSortableShort( - ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue() - ); - long metric4 = HalfFloatPoint.halfFloatToSortableShort( - ((HalfFloatPoint) starTreeDocuments[i].metrics[3]).numericValue().floatValue() - ); - long metric5 = HalfFloatPoint.halfFloatToSortableShort( - ((HalfFloatPoint) starTreeDocuments[i].metrics[4]).numericValue().floatValue() - ); - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(7, resultStarTreeDocuments.size()); - - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); - assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 2, - getExpectedStarTreeDocumentIterator().size(), - 1, - Set.of("field8"), - getBuildMode(), - 0, - 330 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - getExpectedStarTreeDocumentIterator().size(), - starTreeMetadata, - getExpectedStarTreeDocumentIterator() - ); - } - - public void test_build_floatMetrics() throws IOException { - - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.FLOAT, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 12.0F, 10.0F, randomFloat(), 8.0F, 20.0F, null } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 10.0F, 6.0F, randomFloat(), 12.0F, 10.0F, null } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 14.0F, 12.0F, randomFloat(), 6.0F, 24.0F, null } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 9.0F, 4.0F, randomFloat(), 9.0F, 12.0F, null } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 11.0F, 16.0F, randomFloat(), 8.0F, 13.0F, null } - ); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[4]); - Long metric6 = (Long) starTreeDocuments[i].metrics[5]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(7, resultStarTreeDocuments.size()); - - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); - assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 2, - getExpectedStarTreeDocumentIterator().size(), - 1, - Set.of("field8"), - getBuildMode(), - 0, - 330 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - getExpectedStarTreeDocumentIterator().size(), - starTreeMetadata, - getExpectedStarTreeDocumentIterator() - ); - } - - abstract StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode(); - - public void test_build_longMetrics() throws IOException { - - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.LONG, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.LONG, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.LONG, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong(), 8L, 20L }); - starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 10L, 6L, randomLong(), 12L, 10L }); - starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong(), 6L, 24L }); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 9L, 4L, randomLong(), 9L, 12L }); - starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong(), 8L, 13L }); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = (Long) starTreeDocuments[i].metrics[0]; - long metric2 = (Long) starTreeDocuments[i].metrics[1]; - long metric3 = (Long) starTreeDocuments[i].metrics[2]; - long metric4 = (Long) starTreeDocuments[i].metrics[3]; - long metric5 = (Long) starTreeDocuments[i].metrics[4]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, null } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(7, resultStarTreeDocuments.size()); - - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); - assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 2, - getExpectedStarTreeDocumentIterator().size(), - 1, - Set.of("field8"), - getBuildMode(), - 0, - 330 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - getExpectedStarTreeDocumentIterator().size(), - starTreeMetadata, - getExpectedStarTreeDocumentIterator() - ); - } - - private static List getExpectedStarTreeDocumentIterator() { - return List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), - new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), - new StarTreeDocument(new Long[] { null, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), - new StarTreeDocument(new Long[] { null, 4L, null, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), - new StarTreeDocument(new Long[] { null, 4L, null, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), - new StarTreeDocument(new Long[] { null, 4L, null, null }, new Object[] { 56.0, 48.0, 5L, 6.0, 24.0, 5L }) - ); - } - - public void test_build_multipleStarTrees() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 }); - starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 }); - starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 }); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 }); - starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble(), 8.0, 13.0 }); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5 } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - - metrics = List.of( - new Metric("field2", List.of(MetricStat.SUM)), - new Metric("field4", List.of(MetricStat.SUM)), - new Metric("field6", List.of(MetricStat.VALUE_COUNT)), - new Metric("field9", List.of(MetricStat.MIN)), - new Metric("field10", List.of(MetricStat.MAX)) - ); - - compositeField = new StarTreeField( - "test", - dimensionsOrder, - metrics, - new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) - ); - - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(7, resultStarTreeDocuments.size()); - - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); - assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - builder.close(); - - // building another tree in the same file - fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); - - dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); - metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM))); - - DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - - compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); - SegmentInfo segmentInfo = new SegmentInfo( - directory, - Version.LATEST, - Version.LUCENE_9_11_0, - "test_segment", - 7, - false, - false, - new Lucene912Codec(), - new HashMap<>(), - UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), - new HashMap<>(), - null - ); - - fieldsInfo = new FieldInfo[fields.size()]; - fieldProducerMap = new HashMap<>(); - for (int i = 0; i < fieldsInfo.length; i++) { - fieldsInfo[i] = new FieldInfo( - fields.get(i), - i, - false, - false, - true, - IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - fieldProducerMap.put(fields.get(i), docValuesProducer); - } - FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); - - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - - InMemoryTreeNode rootNode1 = builder.getRootNode(); - - int noOfStarTreeDocuments2 = 7; - StarTreeDocument[] starTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; - starTreeDocuments2[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Double[] { 400.0 }); - starTreeDocuments2[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Double[] { 200.0 }); - starTreeDocuments2[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Double[] { 300.0 }); - starTreeDocuments2[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Double[] { 100.0 }); - starTreeDocuments2[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Double[] { 600.0 }); - starTreeDocuments2[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Double[] { 200.0 }); - starTreeDocuments2[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Double[] { 400.0 }); - - StarTreeDocument[] segmentStarTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; - for (int i = 0; i < noOfStarTreeDocuments2; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments2[i].metrics[0]); - segmentStarTreeDocuments2[i] = new StarTreeDocument(starTreeDocuments2[i].dimensions, new Long[] { metric1 }); - } - - SequentialDocValuesIterator[] dimsIterators2 = getDimensionIterators(segmentStarTreeDocuments2); - List metricsIterators2 = getMetricIterators(segmentStarTreeDocuments2); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator2 = builder.sortAndAggregateSegmentDocuments( - dimsIterators2, - metricsIterators2 - ); - builder.build(segmentStarTreeDocumentIterator2, new AtomicInteger(), mock(DocValuesConsumer.class)); - InMemoryTreeNode rootNode2 = builder.getRootNode(); - - metaOut.close(); - dataOut.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3", "field5", "field8"), - List.of( - new Metric("field2", List.of(MetricStat.SUM)), - new Metric("field4", List.of(MetricStat.SUM)), - new Metric("field6", List.of(MetricStat.VALUE_COUNT)), - new Metric("field9", List.of(MetricStat.MIN)), - new Metric("field10", List.of(MetricStat.MAX)) - ), - 2, - getExpectedStarTreeDocumentIterator().size(), - 1, - Set.of("field8"), - getBuildMode(), - 0, - 330 - ); - - StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("fieldC", "fieldB", "fieldL"), - List.of(new Metric("fieldI", List.of(MetricStat.SUM))), - 7, - 27, - 1, - Set.of(), - getBuildMode(), - 330, - 1287 - ); - - List totalDimensionFields = new ArrayList<>(); - totalDimensionFields.addAll(starTreeMetadata.getDimensionFields()); - totalDimensionFields.addAll(starTreeMetadata2.getDimensionFields()); - - List metrics = new ArrayList<>(); - metrics.addAll(starTreeMetadata.getMetrics()); - metrics.addAll(starTreeMetadata2.getMetrics()); - - SegmentReadState readState = getReadState(3, totalDimensionFields, metrics); - - IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); - IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); - - validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata); - validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2); - - dataIn.close(); - metaIn.close(); - - } - - public void test_build() throws IOException { - - int noOfStarTreeDocuments = 5; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - - starTreeDocuments[0] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0, 1L } - ); - starTreeDocuments[1] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, null } - ); - starTreeDocuments[2] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0, null } - ); - starTreeDocuments[3] = new StarTreeDocument( - new Long[] { 2L, 4L, 3L, 4L }, - new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, null } - ); - starTreeDocuments[4] = new StarTreeDocument( - new Long[] { 3L, 4L, 2L, 1L }, - new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, null } - ); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); - long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); - long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); - Long metric6 = (Long) starTreeDocuments[i].metrics[5]; - segmentStarTreeDocuments[i] = new StarTreeDocument( - starTreeDocuments[i].dimensions, - new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } - ); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(7, resultStarTreeDocuments.size()); - - Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); - assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 2, - getExpectedStarTreeDocumentIterator().size(), - 1, - Set.of("field8"), - getBuildMode(), - 0, - 330 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - getExpectedStarTreeDocumentIterator().size(), - starTreeMetadata, - getExpectedStarTreeDocumentIterator() - ); - } - - private void assertStarTreeDocuments( - List resultStarTreeDocuments, - Iterator expectedStarTreeDocumentIterator - ) { - Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); - while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); - assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); - assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); - } - } - - public void test_build_starTreeDataset() throws IOException { - - fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); - - dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); - metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM)), new Metric("_doc_count", List.of(MetricStat.DOC_COUNT))); - - DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - - compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); - SegmentInfo segmentInfo = new SegmentInfo( - directory, - Version.LATEST, - Version.LUCENE_9_11_0, - "test_segment", - 7, - false, - false, - new Lucene912Codec(), - new HashMap<>(), - UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), - new HashMap<>(), - null - ); - - fieldsInfo = new FieldInfo[fields.size()]; - fieldProducerMap = new HashMap<>(); - for (int i = 0; i < fieldsInfo.length; i++) { - fieldsInfo[i] = new FieldInfo( - fields.get(i), - i, - false, - false, - true, - IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false, - false - ); - fieldProducerMap.put(fields.get(i), docValuesProducer); - } - FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - mapperService = mock(MapperService.class); - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) - .build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - - int noOfStarTreeDocuments = 7; - StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, null }); - starTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, null }); - starTreeDocuments[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0, null }); - starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, null }); - starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0, null }); - starTreeDocuments[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0, null }); - starTreeDocuments[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0, null }); - - StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - for (int i = 0; i < noOfStarTreeDocuments; i++) { - long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, null }); - } - - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimsIterators, - metricsIterators - ); - builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - List resultStarTreeDocuments = builder.getStarTreeDocuments(); - Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments().iterator(); - Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); - Map> dimValueToDocIdMap = new HashMap<>(); - builder.rootNode.setNodeType(StarTreeNodeType.STAR.getValue()); - traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); - - Map> expectedDimToValueMap = getExpectedDimToValueMap(); - for (Map.Entry> entry : dimValueToDocIdMap.entrySet()) { - int dimId = entry.getKey(); - if (dimId == -1) continue; - Map map = expectedDimToValueMap.get(dimId); - for (Map.Entry dimValueToDocIdEntry : entry.getValue().entrySet()) { - long dimValue = dimValueToDocIdEntry.getKey(); - int docId = dimValueToDocIdEntry.getValue(); - if (map.get(dimValue) != null) { - assertEquals(map.get(dimValue), resultStarTreeDocuments.get(docId).metrics[0]); - } - } - } - - while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); - StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); - assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); - assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); - assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); - assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); - assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - } - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "test", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 7, - 27, - 1, - Set.of(), - getBuildMode(), - 0, - 1287 - ); - validateStarTreeFileFormats(builder.getRootNode(), 27, starTreeMetadata, expectedStarTreeDocuments()); - } - - private List getStarTreeDimensionNames(List dimensionsOrder) { - - List dimensionNames = new ArrayList<>(); - for (Dimension dimension : dimensionsOrder) { - dimensionNames.addAll(dimension.getSubDimensionNames()); - } - return dimensionNames; - - } - - private void validateStarTreeFileFormats( - InMemoryTreeNode rootNode, - int numDocs, - StarTreeMetadata expectedStarTreeMetadata, - List expectedStarTreeDocuments - ) throws IOException { - - assertNotNull(rootNode.getChildren()); - assertFalse(rootNode.getChildren().isEmpty()); - SegmentReadState readState = getReadState( - numDocs, - expectedStarTreeMetadata.getDimensionFields(), - expectedStarTreeMetadata.getMetrics() - ); - - DocValuesProducer compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( - Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, - readState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - - IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); - IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); - - StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); - assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), starTreeValues.getStarTreeDocumentCount()); - List fieldValueConverters = new ArrayList<>(); - builder.metricAggregatorInfos.forEach( - metricAggregatorInfo -> fieldValueConverters.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) - ); - StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( - List.of(starTreeValues), - fieldValueConverters, - readState.segmentInfo.maxDoc() - ); - - StarTreeDocument[] expectedStarTreeDocumentsArray = expectedStarTreeDocuments.toArray(new StarTreeDocument[0]); - StarTreeTestUtils.assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocumentsArray); - - validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata); - - dataIn.close(); - metaIn.close(); - compositeDocValuesProducer.close(); - } - - private static Map> getExpectedDimToValueMap() { - Map> expectedDimToValueMap = new HashMap<>(); - Map dimValueMap = new HashMap<>(); - dimValueMap.put(1L, 600.0); - dimValueMap.put(2L, 400.0); - dimValueMap.put(3L, 1200.0); - expectedDimToValueMap.put(0, dimValueMap); - - dimValueMap = new HashMap<>(); - dimValueMap.put(11L, 1000.0); - dimValueMap.put(12L, 800.0); - dimValueMap.put(13L, 400.0); - expectedDimToValueMap.put(1, dimValueMap); - - dimValueMap = new HashMap<>(); - dimValueMap.put(21L, 1500.0); - dimValueMap.put(22L, 200.0); - dimValueMap.put(23L, 500.0); - expectedDimToValueMap.put(2, dimValueMap); - return expectedDimToValueMap; - } - - private List expectedStarTreeDocuments() { - return List.of( - new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, 1L }), - new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, 1L }), - new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0, 1L }), - new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0, 1L }), - new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0, 1L }), - new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { null, 11L, 21L }, new Object[] { 1000.0, 2L }), - new StarTreeDocument(new Long[] { null, 12L, 21L }, new Object[] { 400.0, 1L }), - new StarTreeDocument(new Long[] { null, 12L, 22L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { null, 12L, 23L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { null, 13L, 21L }, new Object[] { 100.0, 1L }), - new StarTreeDocument(new Long[] { null, 13L, 23L }, new Object[] { 300.0, 1L }), - new StarTreeDocument(new Long[] { null, null, 21L }, new Object[] { 1500.0, 4L }), - new StarTreeDocument(new Long[] { null, null, 22L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { null, null, 23L }, new Object[] { 500.0, 2L }), - new StarTreeDocument(new Long[] { null, null, null }, new Object[] { 2200.0, 7L }), - new StarTreeDocument(new Long[] { null, 12L, null }, new Object[] { 800.0, 3L }), - new StarTreeDocument(new Long[] { null, 13L, null }, new Object[] { 400.0, 2L }), - new StarTreeDocument(new Long[] { 1L, null, 21L }, new Object[] { 400.0, 1L }), - new StarTreeDocument(new Long[] { 1L, null, 22L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { 1L, null, null }, new Object[] { 600.0, 2L }), - new StarTreeDocument(new Long[] { 2L, 13L, null }, new Object[] { 400.0, 2L }), - new StarTreeDocument(new Long[] { 3L, null, 21L }, new Object[] { 1000.0, 2L }), - new StarTreeDocument(new Long[] { 3L, null, 23L }, new Object[] { 200.0, 1L }), - new StarTreeDocument(new Long[] { 3L, null, null }, new Object[] { 1200.0, 3L }), - new StarTreeDocument(new Long[] { 3L, 12L, null }, new Object[] { 600.0, 2L }) - ); - - } - - public void testFlushFlow() throws IOException { - List dimList = List.of(0L, 1L, 3L, 4L, 5L); - List docsWithField = List.of(0, 1, 3, 4, 5); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); - - List metricsList = List.of( - getLongFromDouble(0.0), - getLongFromDouble(10.0), - getLongFromDouble(20.0), - getLongFromDouble(30.0), - getLongFromDouble(40.0), - getLongFromDouble(50.0) - ); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - - compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); - SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(dimList2, docsWithField2) - ); - SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - - writeState = getWriteState(6, writeState.segmentInfo.getId()); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; - Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimDvs, - List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) - ); - /** - * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] - [0, 0] | [0.0, 1] - [1, 1] | [10.0, 1] - [3, 3] | [30.0, 1] - [4, 4] | [40.0, 1] - [5, 5] | [50.0, 1] - [null, 2] | [20.0, 1] - */ - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - List starTreeDocuments = builder.getStarTreeDocuments(); - int count = 0; - for (StarTreeDocument starTreeDocument : starTreeDocuments) { - count++; - if (starTreeDocument.dimensions[1] != null) { - assertEquals( - starTreeDocument.dimensions[0] == null - ? starTreeDocument.dimensions[1] * 1 * 10.0 - : starTreeDocument.dimensions[0] * 10, - starTreeDocument.metrics[0] - ); - assertEquals(1L, starTreeDocument.metrics[1]); - } else { - assertEquals(150D, starTreeDocument.metrics[0]); - assertEquals(6L, starTreeDocument.metrics[1]); - } - } - assertEquals(13, count); - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 264 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - - } - - public void testFlushFlowDimsReverse() throws IOException { - List dimList = List.of(5L, 4L, 3L, 2L, 1L); - List docsWithField = List.of(0, 1, 2, 3, 4); - List dimList2 = List.of(5L, 4L, 3L, 2L, 1L, 0L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); - - List metricsList = List.of( - getLongFromDouble(50.0), - getLongFromDouble(40.0), - getLongFromDouble(30.0), - getLongFromDouble(20.0), - getLongFromDouble(10.0), - getLongFromDouble(0.0) - ); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - - compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); - SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(dimList2, docsWithField2) - ); - SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - - writeState = getWriteState(6, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; - Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimDvs, - List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) - ); - /** - * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] - [1, 1] | [10.0, 1] - [2, 2] | [20.0, 1] - [3, 3] | [30.0, 1] - [4, 4] | [40.0, 1] - [5, 5] | [50.0, 1] - [null, 0] | [0.0, 1] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(6, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - if (count <= 6) { - count++; - if (starTreeDocument.dimensions[0] != null) { - assertEquals(count, (long) starTreeDocument.dimensions[0]); - } - assertEquals(starTreeDocument.dimensions[1] * 10.0, starTreeDocument.metrics[0]); - assertEquals(1L, starTreeDocument.metrics[1]); - } - } - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 264 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testFlushFlowBuild() throws IOException { - List dimList = new ArrayList<>(100); - List docsWithField = new ArrayList<>(100); - for (int i = 0; i < 100; i++) { - dimList.add((long) i); - docsWithField.add(i); - } - - List dimList2 = new ArrayList<>(100); - List docsWithField2 = new ArrayList<>(100); - for (int i = 0; i < 100; i++) { - dimList2.add((long) i); - docsWithField2.add(i); - } - - List metricsList = new ArrayList<>(100); - List metricsWithField = new ArrayList<>(100); - for (int i = 0; i < 100; i++) { - metricsList.add(getLongFromDouble(i * 10.0)); - metricsWithField.add(i); - } - - Dimension d1 = new NumericDimension("field1"); - Dimension d2 = new NumericDimension("field3"); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); - List dims = List.of(d1, d2); - List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); - compositeField = new StarTreeField("sf", dims, metrics, c); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - - writeState = getWriteState(100, writeState.segmentInfo.getId()); - SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - consumerWriteState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - - DocValuesProducer d1vp = getDocValuesProducer(d1sndv); - DocValuesProducer d2vp = getDocValuesProducer(d2sndv); - DocValuesProducer m1vp = getDocValuesProducer(m1sndv); - Map fieldProducerMap = Map.of("field1", d1vp, "field3", d2vp, "field2", m1vp); - builder.build(fieldProducerMap, new AtomicInteger(), docValuesConsumer); - /** - * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] - [0, 0] | [0.0] - [1, 1] | [10.0] - [2, 2] | [20.0] - [3, 3] | [30.0] - [4, 4] | [40.0] - .... - [null, 0] | [0.0] - [null, 1] | [10.0] - ... - [null, null] | [49500.0] - */ - List starTreeDocuments = builder.getStarTreeDocuments(); - for (StarTreeDocument starTreeDocument : starTreeDocuments) { - assertEquals( - starTreeDocument.dimensions[1] != null ? starTreeDocument.dimensions[1] * 10.0 : 49500.0, - starTreeDocument.metrics[0] - ); - } - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 100, - builder.numStarTreeDocs, - 1, - Set.of(), - getBuildMode(), - 0, - 6699 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - private static DocValuesProducer getDocValuesProducer(SortedNumericDocValues sndv) { - return new EmptyDocValuesProducer() { - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return sndv; - } - }; - } - - private StarTreeField getStarTreeFieldWithMultipleMetrics() { - Dimension d1 = new NumericDimension("field1"); - Dimension d2 = new NumericDimension("field3"); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); - Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); - Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); - List dims = List.of(d1, d2); - List metrics = List.of(m1, m2, m3); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); - return new StarTreeField("sf", dims, metrics, c); - } - - public void testMergeFlow_randomNumberTypes() throws Exception { - - DocumentMapper documentMapper = mock(DocumentMapper.class); - when(mapperService.documentMapper()).thenReturn(documentMapper); - Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); - NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder( - "field1", - randomFrom(NumberFieldMapper.NumberType.values()), - false, - true - ).build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder( - "field2", - randomFrom(NumberFieldMapper.NumberType.values()), - false, - true - ).build(new Mapper.BuilderContext(settings, new ContentPath())); - NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder( - "field3", - randomFrom(NumberFieldMapper.NumberType.values()), - false, - true - ).build(new Mapper.BuilderContext(settings, new ContentPath())); - MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), - Collections.emptyList(), - Collections.emptyList(), - 0, - null - ); - when(documentMapper.mappers()).thenReturn(fieldMappers); - testMergeFlowWithSum(); - } - - public void testMergeFlowWithSum() throws IOException { - List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); - List docsWithField = List.of(0, 1, 3, 4, 5, 6); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of( - getLongFromDouble(0.0), - getLongFromDouble(10.0), - getLongFromDouble(20.0), - getLongFromDouble(30.0), - getLongFromDouble(40.0), - getLongFromDouble(50.0), - getLongFromDouble(60.0) - - ); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - compositeField = getStarTreeField(MetricStat.SUM); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - writeState = getWriteState(6, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] - * [0, 0] | [0.0] - * [1, 1] | [20.0] - * [3, 3] | [60.0] - * [4, 4] | [80.0] - * [5, 5] | [100.0] - * [null, 2] | [40.0] - * ------------------ We only take non star docs - * [6,-1] | [120.0] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(6, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 6) { - assertEquals( - starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, - starTreeDocument.metrics[0] - ); - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 264 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithCount() throws IOException { - List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); - List docsWithField = List.of(0, 1, 3, 4, 5, 6); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - writeState = getWriteState(6, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [2] - [3, 3] | [6] - [4, 4] | [8] - [5, 5] | [10] - [null, 2] | [4] - --------------- - [6,-1] | [12] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(6, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 6) { - assertEquals(starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 : 4, starTreeDocument.metrics[0]); - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 264 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - - } - - private StarTreeValues getStarTreeValues( - SortedNumericDocValues dimList, - SortedNumericDocValues dimList2, - SortedNumericDocValues metricsList, - StarTreeField sf, - String number - ) { - SortedNumericDocValues d1sndv = dimList; - SortedNumericDocValues d2sndv = dimList2; - SortedNumericDocValues m1sndv = metricsList; - Map> dimDocIdSetIterators = Map.of( - "field1", - () -> new SortedNumericStarTreeValuesIterator(d1sndv), - "field3", - () -> new SortedNumericStarTreeValuesIterator(d2sndv) - ); - - Map> metricDocIdSetIterators = new LinkedHashMap<>(); - for (Metric metric : sf.getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { - String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( - sf.getName(), - metric.getField(), - metricStat.getTypeName() - ); - metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); - } - } - - StarTreeValues starTreeValues = new StarTreeValues( - sf, - null, - dimDocIdSetIterators, - metricDocIdSetIterators, - Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number), - null - ); - return starTreeValues; - } - - public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { - List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); - List docsWithField = List.of(0, 1, 3, 4, 5, 6); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - writeState = getWriteState(4, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [3, 3] | [3] - [4, 4] | [4] - [5, 5] | [10] - [6, 6] | [6] - [8, 8] | [8] - [null, 2] | [2] - [null, 7] | [7] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(9, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 9) { - if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { - assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - } - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 9, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 330 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowNumSegmentsDocs() throws IOException { - List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); - List docsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - List dimList3 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 2, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - sf, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - sf, - "4" - ); - builder = getStarTreeBuilder(metaOut, dataOut, sf, getWriteState(4, writeState.segmentInfo.getId()), mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [2, 2] | [2] - [3, 3] | [3] - [4, 4] | [4] - [5, 5] | [10] - [6, 6] | [6] - [7, 7] | [7] - [8, 8] | [8] - */ - int count = 0; - while (starTreeDocumentIterator.hasNext()) { - count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { - assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - assertEquals(9, count); - } - - public void testMergeFlowWithMissingDocs() throws IOException { - List dimList = List.of(0L, 1L, 2L, 3L, 4L, 6L); - List docsWithField = List.of(0, 1, 2, 3, 4, 6); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - writeState = getWriteState(4, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [2, 2] | [2] - [3, 3] | [3] - [4, 4] | [4] - [5, 5] | [5] - [6, 6] | [6] - [8, 8] | [8] - [null, 5] | [5] - [null, 7] | [7] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(10, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 10) { - if (starTreeDocument.dimensions[0] == null) { - assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); - } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 10, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 363 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithMissingDocsWithZero() throws IOException { - List dimList = List.of(0L, 0L, 0L, 0L); - List docsWithField = List.of(0, 1, 2, 6); - List dimList2 = List.of(0L, 0L, 0L, 0L); - List docsWithField2 = List.of(0, 1, 2, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "7" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - writeState = getWriteState(4, writeState.segmentInfo.getId()); - SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - consumerWriteState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [9] - [5, 5] | [5] - [6, 6] | [6] - [8, 8] | [8] - [null, 7] | [7] - [null, null] | [12] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(6, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 6) { - if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { - assertEquals(12L, (long) starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == null) { - assertEquals(7L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == 0) { - assertEquals(9L, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 231 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException { - List dimList = List.of(0L, 0L, 0L, 0L, 0L); - List docsWithField = List.of(0, 1, 2, 6, 8); - List dimList2 = List.of(0L, 0L, 0L, 0L); - List docsWithField2 = List.of(0, 1, 2, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "9" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - writeState = getWriteState(4, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [9] - [0, null] | [8] - [5, 5] | [5] - [6, 6] | [6] - [8, 8] | [8] - [null, 7] | [7] - [null, null] | [19] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(7, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 7) { - if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { - assertEquals(19L, (long) starTreeDocument.metrics[0]); - assertEquals(7, count); - } else if (starTreeDocument.dimensions[0] == null) { - assertEquals(7L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[1] == null) { - assertEquals(8L, starTreeDocument.metrics[0]); - } else if (starTreeDocument.dimensions[0] == 0) { - assertEquals(9L, starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 7, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 231 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 6L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); - List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - writeState = getWriteState(4, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [2, 2] | [2] - [3, 3] | [3] - [4, 4] | [4] - [5, 5] | [5] - [5, null] | [5] - [6, 6] | [6] - [8, 8] | [8] - [null, 7] | [7] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(10, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 10) { - if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[0] == 5) { - assertEquals(starTreeDocument.dimensions[0], starTreeDocument.metrics[0]); - } else { - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 10, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 363 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { - List dimList = List.of(0L, 1L, 2L, 3L, 4L); - List docsWithField = List.of(0, 1, 2, 3, 4); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(5L, 6L, 8L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); - List docsWithField4 = List.of(0, 1, 2, 3, 4); - - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(metricsList2, metricsWithField2), - compositeField, - "4" - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [2, 2] | [2] - [3, 3] | [3] - [4, 4] | [4] - [5, 5] | [5] - [6, 6] | [6] - [8, 8] | [8] - [null, 5] | [5] - [null, 7] | [7] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(10, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 10) { - if (starTreeDocument.dimensions[0] == null) { - assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); - } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 10, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 363 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { - List dimList = List.of(0L, 1L, 2L, 3L, 4L); - List docsWithField = List.of(0, 1, 2, 3, 4); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); - - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - compositeField = getStarTreeField(MetricStat.VALUE_COUNT); - StarTreeValues starTreeValues = getStarTreeValues( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(metricsList, metricsWithField), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - DocValues.emptySortedNumeric(), - DocValues.emptySortedNumeric(), - DocValues.emptySortedNumeric(), - compositeField, - "0" - ); - writeState = getWriteState(0, writeState.segmentInfo.getId()); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] - [0, 0] | [0] - [1, 1] | [1] - [2, 2] | [2] - [3, 3] | [3] - [4, 4] | [4] - [null, 5] | [5] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - assertEquals(6, builder.getStarTreeDocuments().size()); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - int count = 0; - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - if (count <= 6) { - if (starTreeDocument.dimensions[0] == null) { - assertEquals(5L, (long) starTreeDocument.dimensions[1]); - } - assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); - } - } - validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - List.of("field1", "field3"), - compositeField.getMetrics(), - 6, - builder.numStarTreeDocs, - 1000, - Set.of(), - getBuildMode(), - 0, - 264 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithDuplicateDimensionValues() throws IOException { - List dimList1 = new ArrayList<>(500); - List docsWithField1 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 5 + j); - } - } - - List dimList2 = new ArrayList<>(500); - List docsWithField2 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList2.add((long) i); - docsWithField2.add(i * 5 + j); - } - } - - List dimList3 = new ArrayList<>(500); - List docsWithField3 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList3.add((long) i); - docsWithField3.add(i * 5 + j); - } - } - - List dimList4 = new ArrayList<>(500); - List docsWithField4 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList4.add((long) i); - docsWithField4.add(i * 5 + j); - } - } - - List metricsList = new ArrayList<>(100); - List metricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(getLongFromDouble(i * 10.0)); - metricsWithField.add(i); - } - List docCountMetricsList = new ArrayList<>(100); - List docCountMetricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - docCountMetricsList.add(i * 10L); - docCountMetricsWithField.add(i); - } - - compositeField = getStarTreeFieldWithDocCount(1, true); - StarTreeValues starTreeValues = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - docCountMetricsList, - docCountMetricsWithField, - compositeField - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - docCountMetricsList, - docCountMetricsWithField, - compositeField - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); - List starTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(401, starTreeDocuments.size()); - int count = 0; - double sum = 0; - /** - 401 docs get generated - [0, 0, 0, 0] | [200.0, 10] - [1, 1, 1, 1] | [700.0, 10] - [2, 2, 2, 2] | [1200.0, 10] - [3, 3, 3, 3] | [1700.0, 10] - [4, 4, 4, 4] | [2200.0, 10] - ..... - [null, null, null, 99] | [49700.0, 10] - [null, null, null, null] | [2495000.0, 1000] - */ - for (StarTreeDocument starTreeDocument : starTreeDocuments) { - if (starTreeDocument.dimensions[3] == null) { - assertEquals(sum, starTreeDocument.metrics[0]); - assertEquals(2495000L, (long) starTreeDocument.metrics[1]); - } else { - if (starTreeDocument.dimensions[0] != null) { - sum += (double) starTreeDocument.metrics[0]; - } - assertEquals(starTreeDocument.dimensions[3] * 500 + 200.0, starTreeDocument.metrics[0]); - assertEquals(starTreeDocument.dimensions[3] * 500 + 200L, (long) starTreeDocument.metrics[1]); - - } - count++; - } - assertEquals(401, count); - validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 100, - builder.numStarTreeDocs, - 1, - Set.of(), - getBuildMode(), - 0, - 13365 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testMergeFlowWithMaxLeafDocs() throws IOException { - List dimList1 = new ArrayList<>(500); - List docsWithField1 = new ArrayList<>(500); - - for (int i = 0; i < 20; i++) { - for (int j = 0; j < 20; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 20 + j); - } - } - for (int i = 80; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 5 + j); - } - } - List dimList3 = new ArrayList<>(500); - List docsWithField3 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList3.add((long) i); - docsWithField3.add(i * 5 + j); - } - } - List dimList2 = new ArrayList<>(500); - List docsWithField2 = new ArrayList<>(500); - for (int i = 0; i < 10; i++) { - for (int j = 0; j < 50; j++) { - dimList2.add((long) i); - docsWithField2.add(i * 50 + j); - } - } - - List dimList4 = new ArrayList<>(500); - List docsWithField4 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList4.add((long) i); - docsWithField4.add(i * 5 + j); - } - } - - List metricsList = new ArrayList<>(100); - List metricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(getLongFromDouble(i * 10.0)); - metricsWithField.add(i); - } - - List metricsList1 = new ArrayList<>(100); - List metricsWithField1 = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList1.add(1L); - metricsWithField1.add(i); - } - - compositeField = getStarTreeFieldWithDocCount(3, true); - StarTreeValues starTreeValues = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - metricsList1, - metricsWithField1, - compositeField - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - metricsList1, - metricsWithField1, - compositeField - ); - - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); - List starTreeDocuments = builder.getStarTreeDocuments(); - /** - 635 docs get generated - [0, 0, 0, 0] | [200.0, 10] - [0, 0, 1, 1] | [700.0, 10] - [0, 0, 2, 2] | [1200.0, 10] - [0, 0, 3, 3] | [1700.0, 10] - [1, 0, 4, 4] | [2200.0, 10] - [1, 0, 5, 5] | [2700.0, 10] - [1, 0, 6, 6] | [3200.0, 10] - [1, 0, 7, 7] | [3700.0, 10] - [2, 0, 8, 8] | [4200.0, 10] - [2, 0, 9, 9] | [4700.0, 10] - [2, 1, 10, 10] | [5200.0, 10] - [2, 1, 11, 11] | [5700.0, 10] - ..... - [18, 7, null, null] | [147800.0, 40] - ... - [7, 2, null, null] | [28900.0, 20] - ... - [null, null, null, 99] | [49700.0, 10] - ..... - [null, null, null, null] | [2495000.0, 1000] - */ - assertEquals(635, starTreeDocuments.size()); - for (StarTreeDocument starTreeDocument : starTreeDocuments) { - if (starTreeDocument.dimensions[0] != null - && starTreeDocument.dimensions[1] != null - && starTreeDocument.dimensions[2] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[1] != null - && starTreeDocument.dimensions[2] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null - && starTreeDocument.dimensions[2] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null - && starTreeDocument.dimensions[1] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[1] != null) { - assertTrue((long) starTreeDocument.metrics[1] == 20L || (long) starTreeDocument.metrics[1] == 40L); - } else if (starTreeDocument.dimensions[1] != null && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[1] != null) { - assertEquals(100L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null) { - assertEquals(40L, starTreeDocument.metrics[1]); - } - } - validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 100, - builder.numStarTreeDocs, - 3, - Set.of(), - getBuildMode(), - 0, - 23199 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - private StarTreeValues getStarTreeValues( - List dimList1, - List docsWithField1, - List dimList2, - List docsWithField2, - List dimList3, - List docsWithField3, - List dimList4, - List docsWithField4, - List metricsList, - List metricsWithField, - List metricsList1, - List metricsWithField1, - StarTreeField sf - ) { - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); - SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map> dimDocIdSetIterators = Map.of( - "field1", - () -> new SortedNumericStarTreeValuesIterator(d1sndv), - "field3", - () -> new SortedNumericStarTreeValuesIterator(d2sndv), - "field5", - () -> new SortedNumericStarTreeValuesIterator(d3sndv), - "field8", - () -> new SortedNumericStarTreeValuesIterator(d4sndv) - ); - - Map> metricDocIdSetIterators = new LinkedHashMap<>(); - - metricDocIdSetIterators.put( - fullyQualifiedFieldNameForStarTreeMetricsDocValues( - sf.getName(), - "field2", - sf.getMetrics().get(0).getMetrics().get(0).getTypeName() - ), - () -> new SortedNumericStarTreeValuesIterator(m1sndv) - ); - metricDocIdSetIterators.put( - fullyQualifiedFieldNameForStarTreeMetricsDocValues( - sf.getName(), - "_doc_count", - sf.getMetrics().get(1).getMetrics().get(0).getTypeName() - ), - () -> new SortedNumericStarTreeValuesIterator(m2sndv) - ); - // metricDocIdSetIterators.put("field2", () -> m1sndv); - // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); - StarTreeValues starTreeValues = new StarTreeValues( - sf, - null, - dimDocIdSetIterators, - metricDocIdSetIterators, - getAttributes(500), - null - ); - return starTreeValues; - } - - public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOException { - List dimList1 = new ArrayList<>(500); - List docsWithField1 = new ArrayList<>(500); - - for (int i = 0; i < 20; i++) { - for (int j = 0; j < 20; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 20 + j); - } - } - for (int i = 80; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 5 + j); - } - } - List dimList3 = new ArrayList<>(500); - List docsWithField3 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList3.add((long) i); - docsWithField3.add(i * 5 + j); - } - } - List dimList2 = new ArrayList<>(500); - List docsWithField2 = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - dimList2.add((long) 1); - docsWithField2.add(i); - } - - List dimList4 = new ArrayList<>(500); - List docsWithField4 = new ArrayList<>(500); - for (int i = 0; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList4.add((long) i); - docsWithField4.add(i * 5 + j); - } - } - - List metricsList = new ArrayList<>(100); - List metricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(getLongFromDouble(i * 10.0)); - metricsWithField.add(i); - } - - List docCountMetricsList = new ArrayList<>(100); - List docCountMetricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(getLongFromDouble(i * 2)); - metricsWithField.add(i); - } - - compositeField = getStarTreeFieldWithDocCount(3, true); - StarTreeValues starTreeValues = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - docCountMetricsList, - docCountMetricsWithField, - compositeField - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - docCountMetricsList, - docCountMetricsWithField, - compositeField - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); - List starTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(401, starTreeDocuments.size()); - validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 100, - builder.numStarTreeDocs, - compositeField.getStarTreeConfig().maxLeafDocs(), - Set.of(), - getBuildMode(), - 0, - 15345 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public static long getLongFromDouble(double value) { - return NumericUtils.doubleToSortableLong(value); - } - - public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOException { - List dimList1 = new ArrayList<>(500); - List docsWithField1 = new ArrayList<>(500); - Map> expectedDimToValueMap = new HashMap<>(); - Map dimValueMap = new HashMap<>(); - for (int i = 0; i < 20; i++) { - for (int j = 0; j < 20; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 20 + j); - } - // metric = no of docs * 10.0 - dimValueMap.put((long) i, 200.0); - } - for (int i = 80; i < 100; i++) { - for (int j = 0; j < 5; j++) { - dimList1.add((long) i); - docsWithField1.add(i * 5 + j); - } - // metric = no of docs * 10.0 - dimValueMap.put((long) i, 50.0); - } - dimValueMap.put(Long.MAX_VALUE, 5000.0); - expectedDimToValueMap.put(0, dimValueMap); - dimValueMap = new HashMap<>(); - List dimList3 = new ArrayList<>(500); - List docsWithField3 = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - dimList3.add((long) 1); - docsWithField3.add(i); - dimValueMap.put((long) i, 10.0); - } - dimValueMap.put(Long.MAX_VALUE, 5000.0); - expectedDimToValueMap.put(2, dimValueMap); - dimValueMap = new HashMap<>(); - List dimList2 = new ArrayList<>(500); - List docsWithField2 = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - dimList2.add((long) i); - docsWithField2.add(i); - dimValueMap.put((long) i, 10.0); - } - dimValueMap.put(Long.MAX_VALUE, 200.0); - expectedDimToValueMap.put(1, dimValueMap); - dimValueMap = new HashMap<>(); - List dimList4 = new ArrayList<>(500); - List docsWithField4 = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - dimList4.add((long) 1); - docsWithField4.add(i); - dimValueMap.put((long) i, 10.0); - } - dimValueMap.put(Long.MAX_VALUE, 5000.0); - expectedDimToValueMap.put(3, dimValueMap); - List metricsList = new ArrayList<>(100); - List metricsWithField = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(getLongFromDouble(10.0)); - metricsWithField.add(i); - } - List metricsList1 = new ArrayList<>(100); - List metricsWithField1 = new ArrayList<>(100); - for (int i = 0; i < 500; i++) { - metricsList.add(1L); - metricsWithField.add(i); - } - compositeField = getStarTreeFieldWithDocCount(10, true); - StarTreeValues starTreeValues = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - metricsList1, - metricsWithField1, - compositeField - ); - - StarTreeValues starTreeValues2 = getStarTreeValues( - dimList1, - docsWithField1, - dimList2, - docsWithField2, - dimList3, - docsWithField3, - dimList4, - docsWithField4, - metricsList, - metricsWithField, - metricsList1, - metricsWithField1, - compositeField - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); - List starTreeDocuments = builder.getStarTreeDocuments(); - Map> dimValueToDocIdMap = new HashMap<>(); - traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); - for (Map.Entry> entry : dimValueToDocIdMap.entrySet()) { - int dimId = entry.getKey(); - if (dimId == -1) continue; - Map map = expectedDimToValueMap.get(dimId); - for (Map.Entry dimValueToDocIdEntry : entry.getValue().entrySet()) { - long dimValue = dimValueToDocIdEntry.getKey(); - int docId = dimValueToDocIdEntry.getValue(); - assertEquals(map.get(dimValue) * 2, starTreeDocuments.get(docId).metrics[0]); - } - } - assertEquals(1041, starTreeDocuments.size()); - validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 500, - builder.numStarTreeDocs, - compositeField.getStarTreeConfig().maxLeafDocs(), - Set.of(), - getBuildMode(), - 0, - 31779 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - private StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { - Dimension d1 = new NumericDimension("field1"); - Dimension d2 = new NumericDimension("field3"); - Dimension d3 = new NumericDimension("field5"); - Dimension d4 = new NumericDimension("field8"); - List dims = List.of(d1, d2, d3, d4); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); - Metric m2 = null; - if (includeDocCountMetric) { - m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); - } - List metrics = m2 == null ? List.of(m1) : List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(maxLeafDocs, new HashSet<>(), getBuildMode()); - StarTreeField sf = new StarTreeField("sf", dims, metrics, c); - return sf; - } - - private void traverseStarTree(InMemoryTreeNode root, Map> dimValueToDocIdMap, boolean traverStarNodes) { - InMemoryTreeNode starTree = root; - // Use BFS to traverse the star tree - Queue queue = new ArrayDeque<>(); - queue.add(starTree); - int currentDimensionId = -1; - InMemoryTreeNode starTreeNode; - List docIds = new ArrayList<>(); - while ((starTreeNode = queue.poll()) != null) { - int dimensionId = starTreeNode.getDimensionId(); - if (dimensionId > currentDimensionId) { - currentDimensionId = dimensionId; - } - - // store aggregated document of the node - int docId = starTreeNode.getAggregatedDocId(); - Map map = dimValueToDocIdMap.getOrDefault(dimensionId, new HashMap<>()); - if (starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue()) { - map.put(Long.MAX_VALUE, docId); - } else { - map.put(starTreeNode.getDimensionValue(), docId); - } - dimValueToDocIdMap.put(dimensionId, map); - - if (starTreeNode.getChildren() != null - && (!traverStarNodes || starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue())) { - Iterator childrenIterator = starTreeNode.getChildren().values().iterator(); - while (childrenIterator.hasNext()) { - InMemoryTreeNode childNode = childrenIterator.next(); - queue.add(childNode); - } - } - } - } - - public void testMergeFlow() throws IOException { - List dimList1 = new ArrayList<>(1000); - List docsWithField1 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList1.add((long) i); - docsWithField1.add(i); - } - - List dimList2 = new ArrayList<>(1000); - List docsWithField2 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList2.add((long) i); - docsWithField2.add(i); - } - - List dimList3 = new ArrayList<>(1000); - List docsWithField3 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList3.add((long) i); - docsWithField3.add(i); - } - - List dimList4 = new ArrayList<>(1000); - List docsWithField4 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList4.add((long) i); - docsWithField4.add(i); - } - - List dimList5 = new ArrayList<>(1000); - List docsWithField5 = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - dimList5.add((long) i); - docsWithField5.add(i); - } - - List metricsList = new ArrayList<>(1000); - List metricsWithField = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - metricsList.add(getLongFromDouble(i * 10.0)); - metricsWithField.add(i); - } - - List metricsListValueCount = new ArrayList<>(1000); - List metricsWithFieldValueCount = new ArrayList<>(1000); - for (int i = 0; i < 1000; i++) { - metricsListValueCount.add((long) i); - metricsWithFieldValueCount.add(i); - } - - Dimension d1 = new NumericDimension("field1"); - Dimension d2 = new NumericDimension("field3"); - Dimension d3 = new NumericDimension("field5"); - Dimension d4 = new NumericDimension("field8"); - // Dimension d5 = new NumericDimension("field5"); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); - Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); - List dims = List.of(d1, d2, d3, d4); - List metrics = List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); - compositeField = new StarTreeField("sf", dims, metrics, c); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); - SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); - SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map> dimDocIdSetIterators = Map.of( - "field1", - () -> new SortedNumericStarTreeValuesIterator(d1sndv), - "field3", - () -> new SortedNumericStarTreeValuesIterator(d2sndv), - "field5", - () -> new SortedNumericStarTreeValuesIterator(d3sndv), - "field8", - () -> new SortedNumericStarTreeValuesIterator(d4sndv) - ); - - Map> metricDocIdSetIterators = Map.of( - "sf_field2_sum_metric", - () -> new SortedNumericStarTreeValuesIterator(m1sndv), - "sf_field2_value_count_metric", - () -> new SortedNumericStarTreeValuesIterator(valucountsndv), - "sf__doc_count_doc_count_metric", - () -> new SortedNumericStarTreeValuesIterator(m2sndv) - ); - - StarTreeValues starTreeValues = new StarTreeValues( - compositeField, - null, - dimDocIdSetIterators, - metricDocIdSetIterators, - getAttributes(1000), - null - ); - - SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); - SortedNumericDocValues f2d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues f2d3sndv = getSortedNumericMock(dimList3, docsWithField3); - SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); - SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); - SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map> f2dimDocIdSetIterators = Map.of( - "field1", - () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), - "field3", - () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), - "field5", - () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), - "field8", - () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) - ); - - Map> f2metricDocIdSetIterators = Map.of( - "sf_field2_sum_metric", - () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), - "sf_field2_value_count_metric", - () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), - "sf__doc_count_doc_count_metric", - () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) - ); - StarTreeValues starTreeValues2 = new StarTreeValues( - compositeField, - null, - f2dimDocIdSetIterators, - f2metricDocIdSetIterators, - getAttributes(1000), - null - ); - - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - [0, 0, 0, 0] | [0.0, 2] - [1, 1, 1, 1] | [20.0, 2] - [2, 2, 2, 2] | [40.0, 2] - [3, 3, 3, 3] | [60.0, 2] - [4, 4, 4, 4] | [80.0, 2] - [5, 5, 5, 5] | [100.0, 2] - ... - [999, 999, 999, 999] | [19980.0] - */ - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); - assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[1]); - assertEquals(2L, starTreeDocument.metrics[2]); - } - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - - // Validate the star tree structure - validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); - - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - compositeField.getName(), - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 1000, - builder.numStarTreeDocs, - compositeField.getStarTreeConfig().maxLeafDocs(), - Set.of(), - getBuildMode(), - 0, - 132165 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - public void testFlushFlowWithTimestamps() throws IOException { - List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L); - List docsWithField = List.of(0, 1, 3, 4, 5); - List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); - - List metricsList = List.of( - getLongFromDouble(0.0), - getLongFromDouble(10.0), - getLongFromDouble(20.0), - getLongFromDouble(30.0), - getLongFromDouble(40.0), - getLongFromDouble(50.0) - ); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - - compositeField = getStarTreeFieldWithDateDimension(); - SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); - SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(dimList2, docsWithField2) - ); - SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( - getSortedNumericMock(metricsList, metricsWithField) - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(6, writeState.segmentInfo.getId()), mapperService); - SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; - Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( - dimDvs, - List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) - ); - /** - * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] - [1655287920000, 1655287200000, 1655287200000, 4] | [40.0, 1] - [1655287980000, 1655287200000, 1655287200000, 3] | [30.0, 1] - [1655288040000, 1655287200000, 1655287200000, 1] | [10.0, 1] - [1655288040000, 1655287200000, 1655287200000, 5] | [50.0, 1] - [1655288100000, 1655287200000, 1655287200000, 0] | [0.0, 1] - [null, null, null, 2] | [20.0, 1] - */ - int count = 0; - List starTreeDocumentsList = new ArrayList<>(); - starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); - starTreeDocumentIterator = starTreeDocumentsList.iterator(); - while (starTreeDocumentIterator.hasNext()) { - count++; - StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals(starTreeDocument.dimensions[3] * 1 * 10.0, starTreeDocument.metrics[1]); - assertEquals(1L, starTreeDocument.metrics[0]); - } - assertEquals(6, count); - builder.build(starTreeDocumentsList.iterator(), new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); - } - - public void testMergeFlowWithTimestamps() throws IOException { - List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L); - List docsWithField = List.of(0, 1, 2, 3, 4, 6); - List dimList2 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); - List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); - List dimList7 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); - List docsWithField7 = List.of(0, 1, 2, 3, 4, 6); - - List dimList5 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List docsWithField5 = List.of(0, 1, 2, 3, 4, 5, 6); - List metricsList1 = List.of( - getLongFromDouble(0.0), - getLongFromDouble(10.0), - getLongFromDouble(20.0), - getLongFromDouble(30.0), - getLongFromDouble(40.0), - getLongFromDouble(50.0), - getLongFromDouble(60.0) - ); - List metricsWithField1 = List.of(0, 1, 2, 3, 4, 5, 6); - List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); - List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - - List dimList3 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); - List docsWithField3 = List.of(0, 1, 3, 4); - List dimList4 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); - List docsWithField4 = List.of(0, 1, 3, 4); - List dimList8 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); - List docsWithField8 = List.of(0, 1, 3, 4); - - List dimList6 = List.of(5L, 6L, 7L, 8L); - List docsWithField6 = List.of(0, 1, 2, 3); - List metricsList21 = List.of( - getLongFromDouble(50.0), - getLongFromDouble(60.0), - getLongFromDouble(70.0), - getLongFromDouble(80.0), - getLongFromDouble(90.0) - ); - List metricsWithField21 = List.of(0, 1, 2, 3, 4); - List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); - List metricsWithField2 = List.of(0, 1, 2, 3, 4); - - compositeField = getStarTreeFieldWithDateDimension(); - StarTreeValues starTreeValues = getStarTreeValuesWithDates( - getSortedNumericMock(dimList, docsWithField), - getSortedNumericMock(dimList2, docsWithField2), - getSortedNumericMock(dimList7, docsWithField7), - getSortedNumericMock(dimList5, docsWithField5), - getSortedNumericMock(metricsList, metricsWithField), - getSortedNumericMock(metricsList1, metricsWithField1), - compositeField, - "6" - ); - - StarTreeValues starTreeValues2 = getStarTreeValuesWithDates( - getSortedNumericMock(dimList3, docsWithField3), - getSortedNumericMock(dimList4, docsWithField4), - getSortedNumericMock(dimList8, docsWithField8), - getSortedNumericMock(dimList6, docsWithField6), - getSortedNumericMock(metricsList2, metricsWithField2), - getSortedNumericMock(metricsList21, metricsWithField21), - compositeField, - "4" - ); - this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( - writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION - ); - builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); - Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - /** - [1655287972000, 1655287972000, 1655287972000, 3] | [30.0, 3] - [1655288032000, 1655288032000, 1655288032000, 2] | [20.0, 2] - [1655288032000, 1655288032000, 1655288032000, 8] | [80.0, 8] - [1655288092000, 1655288092000, 1655288092000, 1] | [10.0, 1] - [1655288092000, 1655288092000, 1655288092000, 4] | [40.0, 4] - [1655288092000, 1655288092000, 1655288092000, 6] | [60.0, 6] - [1655288152000, 1655288152000, 1655288152000, 0] | [0.0, 0] - [1655288152000, 1655288152000, 1655288152000, 5] | [50.0, 5] - [null, null, null, 5] | [50.0, 5] - [null, null, null, 7] | [70.0, 7] - */ - int count = 0; - builder.appendDocumentsToStarTree(starTreeDocumentIterator); - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { - count++; - assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[1], 0); - assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[0]); - } - assertEquals(10, count); - builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); - validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); - metaOut.close(); - dataOut.close(); - docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", - STAR_TREE, - mock(IndexInput.class), - VERSION_CURRENT, - builder.numStarTreeNodes, - getStarTreeDimensionNames(compositeField.getDimensionsOrder()), - compositeField.getMetrics(), - 10, - builder.numStarTreeDocs, - compositeField.getStarTreeConfig().maxLeafDocs(), - Set.of(), - getBuildMode(), - 0, - 231 - ); - - validateStarTreeFileFormats( - builder.getRootNode(), - builder.getStarTreeDocuments().size(), - starTreeMetadata, - builder.getStarTreeDocuments() - ); - } - - private StarTreeValues getStarTreeValuesWithDates( - SortedNumericDocValues dimList, - SortedNumericDocValues dimList2, - SortedNumericDocValues dimList4, - SortedNumericDocValues dimList3, - SortedNumericDocValues metricsList, - SortedNumericDocValues metricsList1, - StarTreeField sf, - String number - ) { - Map> dimDocIdSetIterators = Map.of( - "field1_minute", - () -> new SortedNumericStarTreeValuesIterator(dimList), - "field1_half-hour", - () -> new SortedNumericStarTreeValuesIterator(dimList4), - "field1_hour", - () -> new SortedNumericStarTreeValuesIterator(dimList2), - "field3", - () -> new SortedNumericStarTreeValuesIterator(dimList3) - ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); - - metricDocIdSetIterators.put( - fullyQualifiedFieldNameForStarTreeMetricsDocValues( - sf.getName(), - "field2", - sf.getMetrics().get(0).getMetrics().get(0).getTypeName() - ), - () -> new SortedNumericStarTreeValuesIterator(metricsList) - ); - metricDocIdSetIterators.put( - fullyQualifiedFieldNameForStarTreeMetricsDocValues( - sf.getName(), - "field2", - sf.getMetrics().get(0).getMetrics().get(1).getTypeName() - ), - () -> new SortedNumericStarTreeValuesIterator(metricsList1) - ); - return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number), null); - } - - private StarTreeField getStarTreeFieldWithDateDimension() { - List intervals = new ArrayList<>(); - intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); - intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); - intervals.add(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); - Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); - Dimension d2 = new NumericDimension("field3"); - Metric m1 = new Metric("field2", List.of(MetricStat.VALUE_COUNT, MetricStat.SUM)); - List dims = List.of(d1, d2); - List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(10, new HashSet<>(), getBuildMode()); - StarTreeField sf = new StarTreeField("sf", dims, metrics, c); - return sf; - } - - private void validateStarTree( - InMemoryTreeNode root, - int totalDimensions, - int maxLeafDocuments, - List starTreeDocuments - ) { - Queue queue = new LinkedList<>(); - queue.offer(new Object[] { root, false }); - while (!queue.isEmpty()) { - Object[] current = queue.poll(); - InMemoryTreeNode node = (InMemoryTreeNode) current[0]; - boolean currentIsStarNode = (boolean) current[1]; - - assertNotNull(node); - - // assert dimensions - if (node.getDimensionId() != StarTreeUtils.ALL) { - assertTrue(node.getDimensionId() >= 0 && node.getDimensionId() < totalDimensions); - } - - if (node.getChildren() != null && !node.getChildren().isEmpty()) { - assertEquals(node.getDimensionId() + 1, node.getChildDimensionId()); - assertTrue(node.getChildDimensionId() < totalDimensions); - InMemoryTreeNode starNode = null; - Object[] nonStarNodeCumulativeMetrics = getMetrics(starTreeDocuments); - for (Map.Entry entry : node.getChildren().entrySet()) { - Long childDimensionValue = entry.getKey(); - InMemoryTreeNode child = entry.getValue(); - Object[] currMetrics = getMetrics(starTreeDocuments); - if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { - // Validate dimension values in documents - for (int i = child.getStartDocId(); i < child.getEndDocId(); i++) { - StarTreeDocument doc = starTreeDocuments.get(i); - int j = 0; - addMetrics(doc, currMetrics, j); - if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { - Long dimension = doc.dimensions[child.getDimensionId()]; - assertEquals(childDimensionValue, dimension); - if (dimension != null) { - assertEquals(child.getDimensionValue(), (long) dimension); - } else { - // TODO : fix this ? - assertEquals(child.getDimensionValue(), StarTreeUtils.ALL); - } - } - } - Object[] aggregatedMetrics = starTreeDocuments.get(child.getAggregatedDocId()).metrics; - int j = 0; - for (Object metric : currMetrics) { - /* - * TODO : refactor this to handle any data type - */ - if (metric instanceof Double) { - nonStarNodeCumulativeMetrics[j] = (double) nonStarNodeCumulativeMetrics[j] + (double) metric; - assertEquals((Double) metric, (Double) aggregatedMetrics[j], 0); - } else if (metric instanceof Long) { - nonStarNodeCumulativeMetrics[j] = (long) nonStarNodeCumulativeMetrics[j] + (long) metric; - assertEquals((long) metric, (long) aggregatedMetrics[j]); - } else if (metric instanceof Float) { - nonStarNodeCumulativeMetrics[j] = (float) nonStarNodeCumulativeMetrics[j] + (float) metric; - assertEquals((float) metric, (float) aggregatedMetrics[j], 0); - } - j++; - } - queue.offer(new Object[] { child, false }); - } else { - starNode = child; - } - } - // Add star node to queue - if (starNode != null) { - Object[] starNodeMetrics = getMetrics(starTreeDocuments); - for (int i = starNode.getStartDocId(); i < starNode.getEndDocId(); i++) { - StarTreeDocument doc = starTreeDocuments.get(i); - int j = 0; - addMetrics(doc, starNodeMetrics, j); - } - int j = 0; - Object[] aggregatedMetrics = starTreeDocuments.get(starNode.getAggregatedDocId()).metrics; - for (Object nonStarNodeCumulativeMetric : nonStarNodeCumulativeMetrics) { - assertEquals(nonStarNodeCumulativeMetric, starNodeMetrics[j]); - assertEquals(starNodeMetrics[j], aggregatedMetrics[j]); - /* - * TODO : refactor this to handle any data type - */ - if (nonStarNodeCumulativeMetric instanceof Double) { - assertEquals((double) nonStarNodeCumulativeMetric, (double) starNodeMetrics[j], 0); - assertEquals((double) nonStarNodeCumulativeMetric, (double) aggregatedMetrics[j], 0); - } else if (nonStarNodeCumulativeMetric instanceof Long) { - assertEquals((long) nonStarNodeCumulativeMetric, (long) starNodeMetrics[j]); - assertEquals((long) nonStarNodeCumulativeMetric, (long) aggregatedMetrics[j]); - } else if (nonStarNodeCumulativeMetric instanceof Float) { - assertEquals((float) nonStarNodeCumulativeMetric, (float) starNodeMetrics[j], 0); - assertEquals((float) nonStarNodeCumulativeMetric, (float) aggregatedMetrics[j], 0); - } - - j++; - } - assertEquals(-1L, starNode.getDimensionValue()); - queue.offer(new Object[] { starNode, true }); - } - } else { - assertTrue(node.getEndDocId() - node.getStartDocId() <= maxLeafDocuments); - } - - if (currentIsStarNode) { - StarTreeDocument prevDoc = null; - int docCount = 0; - int docId = node.getStartDocId(); - int dimensionId = node.getDimensionId(); - - while (docId < node.getEndDocId()) { - StarTreeDocument currentDoc = starTreeDocuments.get(docId); - docCount++; - - // Verify that the dimension at 'dimensionId' is set to STAR_IN_DOC_VALUES_INDEX - assertNull(currentDoc.dimensions[dimensionId]); - - // Verify sorting of documents - if (prevDoc != null) { - assertTrue(compareDocuments(prevDoc, currentDoc, dimensionId + 1, totalDimensions) <= 0); - } - prevDoc = currentDoc; - docId++; - } - - // Verify that the number of generated star documents matches the range in the star node - assertEquals(node.getEndDocId() - node.getStartDocId(), docCount); - } - } - } - - /** - * TODO : refactor this to handle any data type - */ - private static void addMetrics(StarTreeDocument doc, Object[] currMetrics, int j) { - for (Object metric : doc.metrics) { - if (metric instanceof Double) { - currMetrics[j] = (double) currMetrics[j] + (double) metric; - } else if (metric instanceof Long) { - currMetrics[j] = (long) currMetrics[j] + (long) metric; - } else if (metric instanceof Float) { - currMetrics[j] = (float) currMetrics[j] + (float) metric; - } - j++; - } - } - - private static Object[] getMetrics(List starTreeDocuments) { - Object[] nonStarNodeCumulativeMetrics = new Object[starTreeDocuments.get(0).metrics.length]; - for (int i = 0; i < nonStarNodeCumulativeMetrics.length; i++) { - if (starTreeDocuments.get(0).metrics[i] instanceof Long) { - nonStarNodeCumulativeMetrics[i] = 0L; - } else if (starTreeDocuments.get(0).metrics[i] instanceof Double) { - nonStarNodeCumulativeMetrics[i] = 0.0; - } else if (starTreeDocuments.get(0).metrics[i] instanceof Float) { - nonStarNodeCumulativeMetrics[i] = 0.0f; - } - } - return nonStarNodeCumulativeMetrics; - } - - private int compareDocuments(StarTreeDocument doc1, StarTreeDocument doc2, int startDim, int endDim) { - for (int i = startDim; i < endDim; i++) { - Long val1 = doc1.dimensions[i]; - Long val2 = doc2.dimensions[i]; - - if (!Objects.equals(val1, val2)) { - if (val1 == null) return 1; - if (val2 == null) return -1; - return Long.compare(val1, val2); - } - } - return 0; - } - - Map getAttributes(int numSegmentDocs) { - return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); - } - - private StarTreeField getStarTreeField(MetricStat count) { - Dimension d1 = new NumericDimension("field1"); - Dimension d2 = new NumericDimension("field3"); - Metric m1 = new Metric("field2", List.of(count)); - List dims = List.of(d1, d2); - List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); - return new StarTreeField("sf", dims, metrics, c); - } - - SortedNumericDocValues getSortedNumericMock(List dimList, List docsWithField) { - return new SortedNumericDocValues() { - int index = -1; - - @Override - public long nextValue() { - return dimList.get(index); - } - - @Override - public int docValueCount() { - return 0; - } - - @Override - public boolean advanceExact(int target) { - return false; - } - - @Override - public int docID() { - return index; - } - - @Override - public int nextDoc() { - if (index == docsWithField.size() - 1) { - return NO_MORE_DOCS; - } - index++; - return docsWithField.get(index); - } - - @Override - public int advance(int target) { - return 0; - } - - @Override - public long cost() { - return 0; - } - }; - } - - @Override - public void tearDown() throws Exception { - super.tearDown(); - if (builder != null) { - builder.close(); - } - docValuesConsumer.close(); - metaOut.close(); - dataOut.close(); - directory.close(); - } -} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java new file mode 100644 index 0000000000000..bb31bd6a7cc27 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -0,0 +1,526 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; +import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.mapper.FieldValueConverter; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; + +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.apache.lucene.tests.util.LuceneTestCase.newIOContext; +import static org.apache.lucene.tests.util.LuceneTestCase.random; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class BuilderTestsUtils { + public static SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeDocuments) { + SequentialDocValuesIterator[] sequentialDocValuesIterators = + new SequentialDocValuesIterator[starTreeDocuments[0].dimensions.length]; + for (int j = 0; j < starTreeDocuments[0].dimensions.length; j++) { + List dimList = new ArrayList<>(); + List docsWithField = new ArrayList<>(); + + for (int i = 0; i < starTreeDocuments.length; i++) { + if (starTreeDocuments[i].dimensions[j] != null) { + dimList.add(starTreeDocuments[i].dimensions[j]); + docsWithField.add(i); + } + } + sequentialDocValuesIterators[j] = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) + ); + } + return sequentialDocValuesIterators; + } + + public static List getMetricIterators(StarTreeDocument[] starTreeDocuments) { + List sequentialDocValuesIterators = new ArrayList<>(); + for (int j = 0; j < starTreeDocuments[0].metrics.length; j++) { + List metricslist = new ArrayList<>(); + List docsWithField = new ArrayList<>(); + + for (int i = 0; i < starTreeDocuments.length; i++) { + if (starTreeDocuments[i].metrics[j] != null) { + metricslist.add((long) starTreeDocuments[i].metrics[j]); + docsWithField.add(i); + } + } + sequentialDocValuesIterators.add( + new SequentialDocValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) + ); + } + return sequentialDocValuesIterators; + } + + public static SortedNumericDocValues getSortedNumericMock(List dimList, List docsWithField) { + return new SortedNumericDocValues() { + int index = -1; + + @Override + public long nextValue() { + return dimList.get(index); + } + + @Override + public int docValueCount() { + return 0; + } + + @Override + public boolean advanceExact(int target) { + return false; + } + + @Override + public int docID() { + return index; + } + + @Override + public int nextDoc() { + if (index == docsWithField.size() - 1) { + return NO_MORE_DOCS; + } + index++; + return docsWithField.get(index); + } + + @Override + public int advance(int target) { + return 0; + } + + @Override + public long cost() { + return 0; + } + }; + } + + public static void validateStarTree( + InMemoryTreeNode root, + int totalDimensions, + int maxLeafDocuments, + List starTreeDocuments + ) { + Queue queue = new LinkedList<>(); + queue.offer(new Object[] { root, false }); + while (!queue.isEmpty()) { + Object[] current = queue.poll(); + InMemoryTreeNode node = (InMemoryTreeNode) current[0]; + boolean currentIsStarNode = (boolean) current[1]; + + assertNotNull(node); + + // assert dimensions + if (node.getDimensionId() != StarTreeUtils.ALL) { + assertTrue(node.getDimensionId() >= 0 && node.getDimensionId() < totalDimensions); + } + + if (node.getChildren() != null && !node.getChildren().isEmpty()) { + assertEquals(node.getDimensionId() + 1, node.getChildDimensionId()); + assertTrue(node.getChildDimensionId() < totalDimensions); + InMemoryTreeNode starNode = null; + Object[] nonStarNodeCumulativeMetrics = getMetrics(starTreeDocuments); + for (Map.Entry entry : node.getChildren().entrySet()) { + Long childDimensionValue = entry.getKey(); + InMemoryTreeNode child = entry.getValue(); + Object[] currMetrics = getMetrics(starTreeDocuments); + if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { + // Validate dimension values in documents + for (int i = child.getStartDocId(); i < child.getEndDocId(); i++) { + StarTreeDocument doc = starTreeDocuments.get(i); + int j = 0; + addMetrics(doc, currMetrics, j); + if (child.getNodeType() != StarTreeNodeType.STAR.getValue()) { + Long dimension = doc.dimensions[child.getDimensionId()]; + assertEquals(childDimensionValue, dimension); + if (dimension != null) { + assertEquals(child.getDimensionValue(), (long) dimension); + } else { + // TODO : fix this ? + assertEquals(child.getDimensionValue(), StarTreeUtils.ALL); + } + } + } + Object[] aggregatedMetrics = starTreeDocuments.get(child.getAggregatedDocId()).metrics; + int j = 0; + for (Object metric : currMetrics) { + /* + * TODO : refactor this to handle any data type + */ + if (metric instanceof Double) { + nonStarNodeCumulativeMetrics[j] = (double) nonStarNodeCumulativeMetrics[j] + (double) metric; + assertEquals((Double) metric, (Double) aggregatedMetrics[j], 0); + } else if (metric instanceof Long) { + nonStarNodeCumulativeMetrics[j] = (long) nonStarNodeCumulativeMetrics[j] + (long) metric; + assertEquals((long) metric, (long) aggregatedMetrics[j]); + } else if (metric instanceof Float) { + nonStarNodeCumulativeMetrics[j] = (float) nonStarNodeCumulativeMetrics[j] + (float) metric; + assertEquals((float) metric, (float) aggregatedMetrics[j], 0); + } + j++; + } + queue.offer(new Object[] { child, false }); + } else { + starNode = child; + } + } + // Add star node to queue + if (starNode != null) { + Object[] starNodeMetrics = getMetrics(starTreeDocuments); + for (int i = starNode.getStartDocId(); i < starNode.getEndDocId(); i++) { + StarTreeDocument doc = starTreeDocuments.get(i); + int j = 0; + addMetrics(doc, starNodeMetrics, j); + } + int j = 0; + Object[] aggregatedMetrics = starTreeDocuments.get(starNode.getAggregatedDocId()).metrics; + for (Object nonStarNodeCumulativeMetric : nonStarNodeCumulativeMetrics) { + assertEquals(nonStarNodeCumulativeMetric, starNodeMetrics[j]); + assertEquals(starNodeMetrics[j], aggregatedMetrics[j]); + /* + * TODO : refactor this to handle any data type + */ + if (nonStarNodeCumulativeMetric instanceof Double) { + assertEquals((double) nonStarNodeCumulativeMetric, (double) starNodeMetrics[j], 0); + assertEquals((double) nonStarNodeCumulativeMetric, (double) aggregatedMetrics[j], 0); + } else if (nonStarNodeCumulativeMetric instanceof Long) { + assertEquals((long) nonStarNodeCumulativeMetric, (long) starNodeMetrics[j]); + assertEquals((long) nonStarNodeCumulativeMetric, (long) aggregatedMetrics[j]); + } else if (nonStarNodeCumulativeMetric instanceof Float) { + assertEquals((float) nonStarNodeCumulativeMetric, (float) starNodeMetrics[j], 0); + assertEquals((float) nonStarNodeCumulativeMetric, (float) aggregatedMetrics[j], 0); + } + + j++; + } + assertEquals(-1L, starNode.getDimensionValue()); + queue.offer(new Object[] { starNode, true }); + } + } else { + assertTrue(node.getEndDocId() - node.getStartDocId() <= maxLeafDocuments); + } + + if (currentIsStarNode) { + StarTreeDocument prevDoc = null; + int docCount = 0; + int docId = node.getStartDocId(); + int dimensionId = node.getDimensionId(); + + while (docId < node.getEndDocId()) { + StarTreeDocument currentDoc = starTreeDocuments.get(docId); + docCount++; + + // Verify that the dimension at 'dimensionId' is set to STAR_IN_DOC_VALUES_INDEX + assertNull(currentDoc.dimensions[dimensionId]); + + // Verify sorting of documents + if (prevDoc != null) { + assertTrue(compareDocuments(prevDoc, currentDoc, dimensionId + 1, totalDimensions) <= 0); + } + prevDoc = currentDoc; + docId++; + } + + // Verify that the number of generated star documents matches the range in the star node + assertEquals(node.getEndDocId() - node.getStartDocId(), docCount); + } + } + } + + /** + * TODO : refactor this to handle any data type + */ + private static void addMetrics(StarTreeDocument doc, Object[] currMetrics, int j) { + for (Object metric : doc.metrics) { + if (metric instanceof Double) { + currMetrics[j] = (double) currMetrics[j] + (double) metric; + } else if (metric instanceof Long) { + currMetrics[j] = (long) currMetrics[j] + (long) metric; + } else if (metric instanceof Float) { + currMetrics[j] = (float) currMetrics[j] + (float) metric; + } + j++; + } + } + + private static Object[] getMetrics(List starTreeDocuments) { + Object[] nonStarNodeCumulativeMetrics = new Object[starTreeDocuments.get(0).metrics.length]; + for (int i = 0; i < nonStarNodeCumulativeMetrics.length; i++) { + if (starTreeDocuments.get(0).metrics[i] instanceof Long) { + nonStarNodeCumulativeMetrics[i] = 0L; + } else if (starTreeDocuments.get(0).metrics[i] instanceof Double) { + nonStarNodeCumulativeMetrics[i] = 0.0; + } else if (starTreeDocuments.get(0).metrics[i] instanceof Float) { + nonStarNodeCumulativeMetrics[i] = 0.0f; + } + } + return nonStarNodeCumulativeMetrics; + } + + private static int compareDocuments(StarTreeDocument doc1, StarTreeDocument doc2, int startDim, int endDim) { + for (int i = startDim; i < endDim; i++) { + Long val1 = doc1.dimensions[i]; + Long val2 = doc2.dimensions[i]; + + if (!Objects.equals(val1, val2)) { + if (val1 == null) return 1; + if (val2 == null) return -1; + return Long.compare(val1, val2); + } + } + return 0; + } + + public static void validateStarTreeFileFormats( + InMemoryTreeNode rootNode, + int numDocs, + StarTreeMetadata expectedStarTreeMetadata, + List expectedStarTreeDocuments, + String dataFileName, + String metaFileName, + BaseStarTreeBuilder builder, + StarTreeField starTreeField, + SegmentWriteState writeState, + Directory directory + ) throws IOException { + + assertNotNull(rootNode.getChildren()); + assertFalse(rootNode.getChildren().isEmpty()); + SegmentReadState readState = getReadState( + numDocs, + expectedStarTreeMetadata.getDimensionFields(), + expectedStarTreeMetadata.getMetrics(), + starTreeField, + writeState, + directory + ); + + DocValuesProducer compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( + Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + readState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + + IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); + IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + + StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); + assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), starTreeValues.getStarTreeDocumentCount()); + List fieldValueConverters = new ArrayList<>(); + builder.metricAggregatorInfos.forEach( + metricAggregatorInfo -> fieldValueConverters.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) + ); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + fieldValueConverters, + readState.segmentInfo.maxDoc() + ); + + StarTreeDocument[] expectedStarTreeDocumentsArray = expectedStarTreeDocuments.toArray(new StarTreeDocument[0]); + StarTreeTestUtils.assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocumentsArray); + + validateFileFormats(dataIn, metaIn, rootNode, expectedStarTreeMetadata); + + dataIn.close(); + metaIn.close(); + compositeDocValuesProducer.close(); + } + + public static SegmentReadState getReadState( + int numDocs, + List dimensionFields, + List metrics, + StarTreeField compositeField, + SegmentWriteState writeState, + Directory directory + ) { + + int numMetrics = 0; + for (Metric metric : metrics) { + numMetrics += metric.getBaseMetrics().size(); + } + + FieldInfo[] fields = new FieldInfo[dimensionFields.size() + numMetrics]; + + int i = 0; + for (String dimension : dimensionFields) { + fields[i] = new FieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeField.getName(), dimension), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + i++; + } + + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getBaseMetrics()) { + fields[i] = new FieldInfo( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + metric.getField(), + metricStat.getTypeName() + ), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + i++; + } + } + + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + numDocs, + false, + false, + new Lucene912Codec(), + new HashMap<>(), + writeState.segmentInfo.getId(), + new HashMap<>(), + null + ); + return new SegmentReadState(segmentInfo.dir, segmentInfo, new FieldInfos(fields), writeState.context); + } + + static void traverseStarTree(InMemoryTreeNode root, Map> dimValueToDocIdMap, boolean traverStarNodes) { + InMemoryTreeNode starTree = root; + // Use BFS to traverse the star tree + Queue queue = new ArrayDeque<>(); + queue.add(starTree); + int currentDimensionId = -1; + InMemoryTreeNode starTreeNode; + List docIds = new ArrayList<>(); + while ((starTreeNode = queue.poll()) != null) { + int dimensionId = starTreeNode.getDimensionId(); + if (dimensionId > currentDimensionId) { + currentDimensionId = dimensionId; + } + + // store aggregated document of the node + int docId = starTreeNode.getAggregatedDocId(); + Map map = dimValueToDocIdMap.getOrDefault(dimensionId, new HashMap<>()); + if (starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue()) { + map.put(Long.MAX_VALUE, docId); + } else { + map.put(starTreeNode.getDimensionValue(), docId); + } + dimValueToDocIdMap.put(dimensionId, map); + + if (starTreeNode.getChildren() != null + && (!traverStarNodes || starTreeNode.getNodeType() == StarTreeNodeType.STAR.getValue())) { + Iterator childrenIterator = starTreeNode.getChildren().values().iterator(); + while (childrenIterator.hasNext()) { + InMemoryTreeNode childNode = childrenIterator.next(); + queue.add(childNode); + } + } + } + } + + public static SegmentWriteState getWriteState(int numDocs, byte[] id, FieldInfo[] fieldsInfo, Directory directory) { + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + numDocs, + false, + false, + new Lucene912Codec(), + new HashMap<>(), + id, + new HashMap<>(), + null + ); + return new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java deleted file mode 100644 index 496558dbc2e83..0000000000000 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilderTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.builder; - -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.mapper.MapperService; - -import java.io.IOException; - -public class OffHeapStarTreeBuilderTests extends AbstractStarTreeBuilderTests { - @Override - public BaseStarTreeBuilder getStarTreeBuilder( - IndexOutput metaOut, - IndexOutput dataOut, - StarTreeField starTreeField, - SegmentWriteState segmentWriteState, - MapperService mapperService - ) throws IOException { - return new OffHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); - } - - @Override - StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode() { - return StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP; - } - -} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java deleted file mode 100644 index 55cf3bde3cea7..0000000000000 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.builder; - -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.mapper.MapperService; - -import java.io.IOException; - -public class OnHeapStarTreeBuilderTests extends AbstractStarTreeBuilderTests { - - @Override - public BaseStarTreeBuilder getStarTreeBuilder( - IndexOutput metaOut, - IndexOutput dataOut, - StarTreeField starTreeField, - SegmentWriteState segmentWriteState, - MapperService mapperService - ) throws IOException { - return new OnHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); - } - - @Override - StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode() { - return StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP; - } - -} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java new file mode 100644 index 0000000000000..095eda2986b3a --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -0,0 +1,954 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getDimensionIterators; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getMetricIterators; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.traverseStarTree; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StarTreeBuildMetricTests extends StarTreeBuilderTestCase { + + public StarTreeBuildMetricTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + super(buildMode); + } + + public void test_build_halfFloatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder( + "field10", + NumberFieldMapper.NumberType.HALF_FLOAT, + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new HalfFloatPoint[] { + new HalfFloatPoint("hf1", 12), + new HalfFloatPoint("hf6", 10), + new HalfFloatPoint("field6", 10), + new HalfFloatPoint("field9", 8), + new HalfFloatPoint("field10", 20) } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new HalfFloatPoint[] { + new HalfFloatPoint("hf2", 10), + new HalfFloatPoint("hf7", 6), + new HalfFloatPoint("field6", 10), + new HalfFloatPoint("field9", 12), + new HalfFloatPoint("field10", 10) } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new HalfFloatPoint[] { + new HalfFloatPoint("hf3", 14), + new HalfFloatPoint("hf8", 12), + new HalfFloatPoint("field6", 10), + new HalfFloatPoint("field9", 6), + new HalfFloatPoint("field10", 24) } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new HalfFloatPoint[] { + new HalfFloatPoint("hf4", 9), + new HalfFloatPoint("hf9", 4), + new HalfFloatPoint("field6", 10), + new HalfFloatPoint("field9", 9), + new HalfFloatPoint("field10", 12) } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new HalfFloatPoint[] { + new HalfFloatPoint("hf5", 11), + new HalfFloatPoint("hf10", 16), + new HalfFloatPoint("field6", 10), + new HalfFloatPoint("field9", 8), + new HalfFloatPoint("field10", 13) } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[0]).numericValue().floatValue() + ); + long metric2 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() + ); + long metric3 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue() + ); + long metric4 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[3]).numericValue().floatValue() + ); + long metric5 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[4]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); + } + + public void test_build_floatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 12.0F, 10.0F, randomFloat(), 8.0F, 20.0F, null } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 10.0F, 6.0F, randomFloat(), 12.0F, 10.0F, null } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 14.0F, 12.0F, randomFloat(), 6.0F, 24.0F, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 9.0F, 4.0F, randomFloat(), 9.0F, 12.0F, null } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 11.0F, 16.0F, randomFloat(), 8.0F, 13.0F, null } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[4]); + Long metric6 = (Long) starTreeDocuments[i].metrics[5]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); + } + + public void test_build_longMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong(), 8L, 20L }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 10L, 6L, randomLong(), 12L, 10L }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong(), 6L, 24L }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 9L, 4L, randomLong(), 9L, 12L }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong(), 8L, 13L }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = (Long) starTreeDocuments[i].metrics[0]; + long metric2 = (Long) starTreeDocuments[i].metrics[1]; + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + long metric4 = (Long) starTreeDocuments[i].metrics[3]; + long metric5 = (Long) starTreeDocuments[i].metrics[4]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); + } + + public void test_build_multipleStarTrees() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble(), 8.0, 13.0 }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5 } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.VALUE_COUNT)), + new Metric("field9", List.of(MetricStat.MIN)), + new Metric("field10", List.of(MetricStat.MAX)) + ); + + compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) + ); + + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + builder.close(); + + // building another tree in the same file + fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); + + dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); + metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM))); + + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 7, + false, + false, + new Lucene912Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + fieldProducerMap = new HashMap<>(); + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + fieldProducerMap.put(fields.get(i), docValuesProducer); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + InMemoryTreeNode rootNode1 = builder.getRootNode(); + + int noOfStarTreeDocuments2 = 7; + StarTreeDocument[] starTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; + starTreeDocuments2[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Double[] { 400.0 }); + starTreeDocuments2[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Double[] { 200.0 }); + starTreeDocuments2[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Double[] { 300.0 }); + starTreeDocuments2[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Double[] { 100.0 }); + starTreeDocuments2[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Double[] { 600.0 }); + starTreeDocuments2[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Double[] { 200.0 }); + starTreeDocuments2[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Double[] { 400.0 }); + + StarTreeDocument[] segmentStarTreeDocuments2 = new StarTreeDocument[noOfStarTreeDocuments2]; + for (int i = 0; i < noOfStarTreeDocuments2; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments2[i].metrics[0]); + segmentStarTreeDocuments2[i] = new StarTreeDocument(starTreeDocuments2[i].dimensions, new Long[] { metric1 }); + } + + SequentialDocValuesIterator[] dimsIterators2 = getDimensionIterators(segmentStarTreeDocuments2); + List metricsIterators2 = getMetricIterators(segmentStarTreeDocuments2); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator2 = builder.sortAndAggregateSegmentDocuments( + dimsIterators2, + metricsIterators2 + ); + builder.build(segmentStarTreeDocumentIterator2, new AtomicInteger(), mock(DocValuesConsumer.class)); + InMemoryTreeNode rootNode2 = builder.getRootNode(); + + metaOut.close(); + dataOut.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3", "field5", "field8"), + List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.VALUE_COUNT)), + new Metric("field9", List.of(MetricStat.MIN)), + new Metric("field10", List.of(MetricStat.MAX)) + ), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("fieldC", "fieldB", "fieldL"), + List.of(new Metric("fieldI", List.of(MetricStat.SUM))), + 7, + 27, + 1, + Set.of(), + getBuildMode(), + 330, + 1287 + ); + + List totalDimensionFields = new ArrayList<>(); + totalDimensionFields.addAll(starTreeMetadata.getDimensionFields()); + totalDimensionFields.addAll(starTreeMetadata2.getDimensionFields()); + + List metrics = new ArrayList<>(); + metrics.addAll(starTreeMetadata.getMetrics()); + metrics.addAll(starTreeMetadata2.getMetrics()); + + SegmentReadState readState = getReadState(3, totalDimensionFields, metrics); + + IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); + IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); + + validateFileFormats(dataIn, metaIn, rootNode1, starTreeMetadata); + validateFileFormats(dataIn, metaIn, rootNode2, starTreeMetadata2); + + dataIn.close(); + metaIn.close(); + + } + + public void test_build() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0, 1L } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, null } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, null } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, null } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = (Long) starTreeDocuments[i].metrics[5]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(7, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator().iterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 2, + getExpectedStarTreeDocumentIterator().size(), + 1, + Set.of("field8"), + getBuildMode(), + 0, + 330 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + getExpectedStarTreeDocumentIterator().size(), + starTreeMetadata, + getExpectedStarTreeDocumentIterator() + ); + } + + public void test_build_starTreeDataset() throws IOException { + + fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); + + dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); + metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM)), new Metric("_doc_count", List.of(MetricStat.DOC_COUNT))); + + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + compositeField = new StarTreeField("test", dimensionsOrder, metrics, new StarTreeFieldConfiguration(1, Set.of(), getBuildMode())); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 7, + false, + false, + new Lucene912Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + fieldProducerMap = new HashMap<>(); + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + fieldProducerMap.put(fields.get(i), docValuesProducer); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + int noOfStarTreeDocuments = 7; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, null }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, null }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0, null }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, null }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0, null }); + starTreeDocuments[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0, null }); + starTreeDocuments[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0, null }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, null }); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments().iterator(); + Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); + Map> dimValueToDocIdMap = new HashMap<>(); + builder.rootNode.setNodeType(StarTreeNodeType.STAR.getValue()); + traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); + + Map> expectedDimToValueMap = getExpectedDimToValueMap(); + for (Map.Entry> entry : dimValueToDocIdMap.entrySet()) { + int dimId = entry.getKey(); + if (dimId == -1) continue; + Map map = expectedDimToValueMap.get(dimId); + for (Map.Entry dimValueToDocIdEntry : entry.getValue().entrySet()) { + long dimValue = dimValueToDocIdEntry.getKey(); + int docId = dimValueToDocIdEntry.getValue(); + if (map.get(dimValue) != null) { + assertEquals(map.get(dimValue), resultStarTreeDocuments.get(docId).metrics[0]); + } + } + } + + while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + } + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "test", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 7, + 27, + 1, + Set.of(), + getBuildMode(), + 0, + 1287 + ); + validateStarTreeFileFormats(builder.getRootNode(), 27, starTreeMetadata, expectedStarTreeDocuments()); + } + + private static Map> getExpectedDimToValueMap() { + Map> expectedDimToValueMap = new HashMap<>(); + Map dimValueMap = new HashMap<>(); + dimValueMap.put(1L, 600.0); + dimValueMap.put(2L, 400.0); + dimValueMap.put(3L, 1200.0); + expectedDimToValueMap.put(0, dimValueMap); + + dimValueMap = new HashMap<>(); + dimValueMap.put(11L, 1000.0); + dimValueMap.put(12L, 800.0); + dimValueMap.put(13L, 400.0); + expectedDimToValueMap.put(1, dimValueMap); + + dimValueMap = new HashMap<>(); + dimValueMap.put(21L, 1500.0); + dimValueMap.put(22L, 200.0); + dimValueMap.put(23L, 500.0); + expectedDimToValueMap.put(2, dimValueMap); + return expectedDimToValueMap; + } + + private List expectedStarTreeDocuments() { + return List.of( + new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0, 1L }), + new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0, 1L }), + new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0, 1L }), + new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0, 1L }), + new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0, 1L }), + new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { null, 11L, 21L }, new Object[] { 1000.0, 2L }), + new StarTreeDocument(new Long[] { null, 12L, 21L }, new Object[] { 400.0, 1L }), + new StarTreeDocument(new Long[] { null, 12L, 22L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { null, 12L, 23L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { null, 13L, 21L }, new Object[] { 100.0, 1L }), + new StarTreeDocument(new Long[] { null, 13L, 23L }, new Object[] { 300.0, 1L }), + new StarTreeDocument(new Long[] { null, null, 21L }, new Object[] { 1500.0, 4L }), + new StarTreeDocument(new Long[] { null, null, 22L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { null, null, 23L }, new Object[] { 500.0, 2L }), + new StarTreeDocument(new Long[] { null, null, null }, new Object[] { 2200.0, 7L }), + new StarTreeDocument(new Long[] { null, 12L, null }, new Object[] { 800.0, 3L }), + new StarTreeDocument(new Long[] { null, 13L, null }, new Object[] { 400.0, 2L }), + new StarTreeDocument(new Long[] { 1L, null, 21L }, new Object[] { 400.0, 1L }), + new StarTreeDocument(new Long[] { 1L, null, 22L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { 1L, null, null }, new Object[] { 600.0, 2L }), + new StarTreeDocument(new Long[] { 2L, 13L, null }, new Object[] { 400.0, 2L }), + new StarTreeDocument(new Long[] { 3L, null, 21L }, new Object[] { 1000.0, 2L }), + new StarTreeDocument(new Long[] { 3L, null, 23L }, new Object[] { 200.0, 1L }), + new StarTreeDocument(new Long[] { 3L, null, null }, new Object[] { 1200.0, 3L }), + new StarTreeDocument(new Long[] { 3L, 12L, null }, new Object[] { 600.0, 2L }) + ); + } + +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java new file mode 100644 index 0000000000000..1aa830e3587df --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -0,0 +1,419 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.EmptyDocValuesProducer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexInput; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedNumericMock; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.mockito.Mockito.mock; + +public class StarTreeBuilderFlushFlowTests extends StarTreeBuilderTestCase { + + public StarTreeBuilderFlushFlowTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + super(buildMode); + } + + public void testFlushFlow() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithMultipleMetrics(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + + writeState = getWriteState(6, writeState.segmentInfo.getId()); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [0, 0] | [0.0, 1] + [1, 1] | [10.0, 1] + [3, 3] | [30.0, 1] + [4, 4] | [40.0, 1] + [5, 5] | [50.0, 1] + [null, 2] | [20.0, 1] + */ + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + int count = 0; + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + count++; + if (starTreeDocument.dimensions[1] != null) { + assertEquals( + starTreeDocument.dimensions[0] == null + ? starTreeDocument.dimensions[1] * 1 * 10.0 + : starTreeDocument.dimensions[0] * 10, + starTreeDocument.metrics[0] + ); + assertEquals(1L, starTreeDocument.metrics[1]); + } else { + assertEquals(150D, starTreeDocument.metrics[0]); + assertEquals(6L, starTreeDocument.metrics[1]); + } + } + assertEquals(13, count); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + + } + + public void testFlushFlowDimsReverse() throws IOException { + List dimList = List.of(5L, 4L, 3L, 2L, 1L); + List docsWithField = List.of(0, 1, 2, 3, 4); + List dimList2 = List.of(5L, 4L, 3L, 2L, 1L, 0L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(50.0), + getLongFromDouble(40.0), + getLongFromDouble(30.0), + getLongFromDouble(20.0), + getLongFromDouble(10.0), + getLongFromDouble(0.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithMultipleMetrics(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [1, 1] | [10.0, 1] + [2, 2] | [20.0, 1] + [3, 3] | [30.0, 1] + [4, 4] | [40.0, 1] + [5, 5] | [50.0, 1] + [null, 0] | [0.0, 1] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + if (count <= 6) { + count++; + if (starTreeDocument.dimensions[0] != null) { + assertEquals(count, (long) starTreeDocument.dimensions[0]); + } + assertEquals(starTreeDocument.dimensions[1] * 10.0, starTreeDocument.metrics[0]); + assertEquals(1L, starTreeDocument.metrics[1]); + } + } + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + List.of("field1", "field3"), + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testFlushFlowBuild() throws IOException { + List dimList = new ArrayList<>(100); + List docsWithField = new ArrayList<>(100); + for (int i = 0; i < 100; i++) { + dimList.add((long) i); + docsWithField.add(i); + } + + List dimList2 = new ArrayList<>(100); + List docsWithField2 = new ArrayList<>(100); + for (int i = 0; i < 100; i++) { + dimList2.add((long) i); + docsWithField2.add(i); + } + + List metricsList = new ArrayList<>(100); + List metricsWithField = new ArrayList<>(100); + for (int i = 0; i < 100; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + Dimension d1 = new NumericDimension("field1"); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + + writeState = getWriteState(100, writeState.segmentInfo.getId()); + SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + consumerWriteState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + + DocValuesProducer d1vp = getDocValuesProducer(d1sndv); + DocValuesProducer d2vp = getDocValuesProducer(d2sndv); + DocValuesProducer m1vp = getDocValuesProducer(m1sndv); + Map fieldProducerMap = Map.of("field1", d1vp, "field3", d2vp, "field2", m1vp); + builder.build(fieldProducerMap, new AtomicInteger(), docValuesConsumer); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] + [0, 0] | [0.0] + [1, 1] | [10.0] + [2, 2] | [20.0] + [3, 3] | [30.0] + [4, 4] | [40.0] + .... + [null, 0] | [0.0] + [null, 1] | [10.0] + ... + [null, null] | [49500.0] + */ + List starTreeDocuments = builder.getStarTreeDocuments(); + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + assertEquals( + starTreeDocument.dimensions[1] != null ? starTreeDocument.dimensions[1] * 10.0 : 49500.0, + starTreeDocument.metrics[0] + ); + } + validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 100, 1, 6699); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testFlushFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithDateDimension(); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(6, writeState.segmentInfo.getId()), mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [1655287920000, 1655287200000, 1655287200000, 4] | [40.0, 1] + [1655287980000, 1655287200000, 1655287200000, 3] | [30.0, 1] + [1655288040000, 1655287200000, 1655287200000, 1] | [10.0, 1] + [1655288040000, 1655287200000, 1655287200000, 5] | [50.0, 1] + [1655288100000, 1655287200000, 1655287200000, 0] | [0.0, 1] + [null, null, null, 2] | [20.0, 1] + */ + int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + starTreeDocumentIterator = starTreeDocumentsList.iterator(); + while (starTreeDocumentIterator.hasNext()) { + count++; + StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); + assertEquals(starTreeDocument.dimensions[3] * 1 * 10.0, starTreeDocument.metrics[1]); + assertEquals(1L, starTreeDocument.metrics[0]); + } + assertEquals(6, count); + builder.build(starTreeDocumentsList.iterator(), new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); + } + + private StarTreeField getStarTreeFieldWithMultipleMetrics() { + Dimension d1 = new NumericDimension("field1"); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); + List dims = List.of(d1, d2); + List metrics = List.of(m1, m2, m3); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + + private static DocValuesProducer getDocValuesProducer(SortedNumericDocValues sndv) { + return new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + return sndv; + } + }; + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java new file mode 100644 index 0000000000000..f983365dfec30 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -0,0 +1,1921 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedNumericMock; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.traverseStarTree; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StarTreeBuilderMergeFlowTests extends StarTreeBuilderTestCase { + + public StarTreeBuilderMergeFlowTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + super(buildMode); + } + + public void testMergeFlow() throws IOException { + List dimList1 = new ArrayList<>(1000); + List docsWithField1 = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + dimList1.add((long) i); + docsWithField1.add(i); + } + + List dimList2 = new ArrayList<>(1000); + List docsWithField2 = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + dimList2.add((long) i); + docsWithField2.add(i); + } + + List dimList3 = new ArrayList<>(1000); + List docsWithField3 = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + dimList3.add((long) i); + docsWithField3.add(i); + } + + List dimList4 = new ArrayList<>(1000); + List docsWithField4 = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + dimList4.add((long) i); + docsWithField4.add(i); + } + + List dimList5 = new ArrayList<>(1000); + List docsWithField5 = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + dimList5.add((long) i); + docsWithField5.add(i); + } + + List metricsList = new ArrayList<>(1000); + List metricsWithField = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + List metricsListValueCount = new ArrayList<>(1000); + List metricsWithFieldValueCount = new ArrayList<>(1000); + for (int i = 0; i < 1000; i++) { + metricsListValueCount.add((long) i); + metricsWithFieldValueCount.add(i); + } + + Dimension d1 = new NumericDimension("field1"); + Dimension d2 = new NumericDimension("field3"); + Dimension d3 = new NumericDimension("field5"); + Dimension d4 = new NumericDimension("field8"); + // Dimension d5 = new NumericDimension("field5"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.AVG, MetricStat.VALUE_COUNT)); + Metric m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); + List dims = List.of(d1, d2, d3, d4); + List metrics = List.of(m1, m2); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + compositeField = new StarTreeField("sf", dims, metrics, c); + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(d4sndv) + ); + + Map> metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(m2sndv) + ); + + StarTreeValues starTreeValues = new StarTreeValues( + compositeField, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(1000), + null + ); + + SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues f2d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues f2d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues f2d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); + SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); + Map> f2dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) + ); + + Map> f2metricDocIdSetIterators = Map.of( + "sf_field2_sum_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), + "sf_field2_value_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), + "sf__doc_count_doc_count_metric", + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) + ); + StarTreeValues starTreeValues2 = new StarTreeValues( + compositeField, + null, + f2dimDocIdSetIterators, + f2metricDocIdSetIterators, + getAttributes(1000), + null + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [0, 0, 0, 0] | [0.0, 2] + [1, 1, 1, 1] | [20.0, 2] + [2, 2, 2, 2] | [40.0, 2] + [3, 3, 3, 3] | [60.0, 2] + [4, 4, 4, 4] | [80.0, 2] + [5, 5, 5, 5] | [100.0, 2] + ... + [999, 999, 999, 999] | [19980.0] + */ + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); + assertEquals(2L, starTreeDocument.metrics[1]); + } + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + // Validate the star tree structure + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 1000, + compositeField.getStarTreeConfig().maxLeafDocs(), + 132165 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlow_randomNumberTypes() throws Exception { + + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder( + "field1", + randomFrom(NumberFieldMapper.NumberType.values()), + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder( + "field2", + randomFrom(NumberFieldMapper.NumberType.values()), + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder( + "field3", + randomFrom(NumberFieldMapper.NumberType.values()), + false, + true + ).build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + testMergeFlowWithSum(); + } + + public void testMergeFlowWithSum() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + compositeField = getStarTreeField(MetricStat.SUM); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] + * [0, 0] | [0.0] + * [1, 1] | [20.0] + * [3, 3] | [60.0] + * [4, 4] | [80.0] + * [5, 5] | [100.0] + * [null, 2] | [40.0] + * ------------------ We only take non star docs + * [6,-1] | [120.0] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 6) { + assertEquals( + starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 * 10.0 : 40.0, + starTreeDocument.metrics[0] + ); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithCount() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + writeState = getWriteState(6, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [2] + [3, 3] | [6] + [4, 4] | [8] + [5, 5] | [10] + [null, 2] | [4] + --------------- + [6,-1] | [12] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 6) { + assertEquals(starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 : 4, starTreeDocument.metrics[0]); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + + } + + public void testMergeFlowNumSegmentsDocs() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); + List docsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, -1L, -1L, -1L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + List dimList3 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 2, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + sf, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + sf, + "4" + ); + builder = getStarTreeBuilder(metaOut, dataOut, sf, getWriteState(4, writeState.segmentInfo.getId()), mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [2, 2] | [2] + [3, 3] | [3] + [4, 4] | [4] + [5, 5] | [10] + [6, 6] | [6] + [7, 7] | [7] + [8, 8] | [8] + */ + int count = 0; + while (starTreeDocumentIterator.hasNext()) { + count++; + StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); + if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { + assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + assertEquals(9, count); + } + + public void testMergeFlowWithMissingDocs() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L, 6L); + List docsWithField = List.of(0, 1, 2, 3, 4, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [2, 2] | [2] + [3, 3] | [3] + [4, 4] | [4] + [5, 5] | [5] + [6, 6] | [6] + [8, 8] | [8] + [null, 5] | [5] + [null, 7] | [7] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 10) { + if (starTreeDocument.dimensions[0] == null) { + assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithMissingDocsWithZero() throws IOException { + List dimList = List.of(0L, 0L, 0L, 0L); + List docsWithField = List.of(0, 1, 2, 6); + List dimList2 = List.of(0L, 0L, 0L, 0L); + List docsWithField2 = List.of(0, 1, 2, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "7" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + consumerWriteState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [9] + [5, 5] | [5] + [6, 6] | [6] + [8, 8] | [8] + [null, 7] | [7] + [null, null] | [12] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 6) { + if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { + assertEquals(12L, (long) starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == null) { + assertEquals(7L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == 0) { + assertEquals(9L, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 231); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException { + List dimList = List.of(0L, 0L, 0L, 0L, 0L); + List docsWithField = List.of(0, 1, 2, 6, 8); + List dimList2 = List.of(0L, 0L, 0L, 0L); + List docsWithField2 = List.of(0, 1, 2, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "9" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [9] + [0, null] | [8] + [5, 5] | [5] + [6, 6] | [6] + [8, 8] | [8] + [null, 7] | [7] + [null, null] | [19] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(7, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 7) { + if (starTreeDocument.dimensions[0] == null && starTreeDocument.dimensions[1] == null) { + assertEquals(19L, (long) starTreeDocument.metrics[0]); + assertEquals(7, count); + } else if (starTreeDocument.dimensions[0] == null) { + assertEquals(7L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[1] == null) { + assertEquals(8L, starTreeDocument.metrics[0]); + } else if (starTreeDocument.dimensions[0] == 0) { + assertEquals(9L, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 7, 1000, 231); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 6L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); + List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [2, 2] | [2] + [3, 3] | [3] + [4, 4] | [4] + [5, 5] | [5] + [5, null] | [5] + [6, 6] | [6] + [8, 8] | [8] + [null, 7] | [7] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 10) { + if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[0] == 5) { + assertEquals(starTreeDocument.dimensions[0], starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L); + List docsWithField = List.of(0, 1, 2, 3, 4); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [2, 2] | [2] + [3, 3] | [3] + [4, 4] | [4] + [5, 5] | [5] + [6, 6] | [6] + [8, 8] | [8] + [null, 5] | [5] + [null, 7] | [7] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(10, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 10) { + if (starTreeDocument.dimensions[0] == null) { + assertTrue(List.of(5L, 7L).contains(starTreeDocument.dimensions[1])); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L); + List docsWithField = List.of(0, 1, 2, 3, 4); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + DocValues.emptySortedNumeric(), + DocValues.emptySortedNumeric(), + DocValues.emptySortedNumeric(), + compositeField, + "0" + ); + writeState = getWriteState(0, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [2, 2] | [2] + [3, 3] | [3] + [4, 4] | [4] + [null, 5] | [5] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(6, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 6) { + if (starTreeDocument.dimensions[0] == null) { + assertEquals(5L, (long) starTreeDocument.dimensions[1]); + } + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithDuplicateDimensionValues() throws IOException { + List dimList1 = new ArrayList<>(500); + List docsWithField1 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 5 + j); + } + } + + List dimList2 = new ArrayList<>(500); + List docsWithField2 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList2.add((long) i); + docsWithField2.add(i * 5 + j); + } + } + + List dimList3 = new ArrayList<>(500); + List docsWithField3 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList3.add((long) i); + docsWithField3.add(i * 5 + j); + } + } + + List dimList4 = new ArrayList<>(500); + List docsWithField4 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList4.add((long) i); + docsWithField4.add(i * 5 + j); + } + } + + List metricsList = new ArrayList<>(100); + List metricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + List docCountMetricsList = new ArrayList<>(100); + List docCountMetricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + docCountMetricsList.add(i * 10L); + docCountMetricsWithField.add(i); + } + + compositeField = getStarTreeFieldWithDocCount(1, true); + StarTreeValues starTreeValues = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + docCountMetricsList, + docCountMetricsWithField, + compositeField + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + docCountMetricsList, + docCountMetricsWithField, + compositeField + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(401, starTreeDocuments.size()); + int count = 0; + double sum = 0; + /** + 401 docs get generated + [0, 0, 0, 0] | [200.0, 10] + [1, 1, 1, 1] | [700.0, 10] + [2, 2, 2, 2] | [1200.0, 10] + [3, 3, 3, 3] | [1700.0, 10] + [4, 4, 4, 4] | [2200.0, 10] + ..... + [null, null, null, 99] | [49700.0, 10] + [null, null, null, null] | [2495000.0, 1000] + */ + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + if (starTreeDocument.dimensions[3] == null) { + assertEquals(sum, starTreeDocument.metrics[0]); + assertEquals(2495000L, (long) starTreeDocument.metrics[1]); + } else { + if (starTreeDocument.dimensions[0] != null) { + sum += (double) starTreeDocument.metrics[0]; + } + assertEquals(starTreeDocument.dimensions[3] * 500 + 200.0, starTreeDocument.metrics[0]); + assertEquals(starTreeDocument.dimensions[3] * 500 + 200L, (long) starTreeDocument.metrics[1]); + + } + count++; + } + assertEquals(401, count); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 100, + 1, + 13365 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithMaxLeafDocs() throws IOException { + List dimList1 = new ArrayList<>(500); + List docsWithField1 = new ArrayList<>(500); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 20; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 20 + j); + } + } + for (int i = 80; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 5 + j); + } + } + List dimList3 = new ArrayList<>(500); + List docsWithField3 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList3.add((long) i); + docsWithField3.add(i * 5 + j); + } + } + List dimList2 = new ArrayList<>(500); + List docsWithField2 = new ArrayList<>(500); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 50; j++) { + dimList2.add((long) i); + docsWithField2.add(i * 50 + j); + } + } + + List dimList4 = new ArrayList<>(500); + List docsWithField4 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList4.add((long) i); + docsWithField4.add(i * 5 + j); + } + } + + List metricsList = new ArrayList<>(100); + List metricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + List metricsList1 = new ArrayList<>(100); + List metricsWithField1 = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList1.add(1L); + metricsWithField1.add(i); + } + + compositeField = getStarTreeFieldWithDocCount(3, true); + StarTreeValues starTreeValues = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + metricsList1, + metricsWithField1, + compositeField + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + metricsList1, + metricsWithField1, + compositeField + ); + + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + /** + 635 docs get generated + [0, 0, 0, 0] | [200.0, 10] + [0, 0, 1, 1] | [700.0, 10] + [0, 0, 2, 2] | [1200.0, 10] + [0, 0, 3, 3] | [1700.0, 10] + [1, 0, 4, 4] | [2200.0, 10] + [1, 0, 5, 5] | [2700.0, 10] + [1, 0, 6, 6] | [3200.0, 10] + [1, 0, 7, 7] | [3700.0, 10] + [2, 0, 8, 8] | [4200.0, 10] + [2, 0, 9, 9] | [4700.0, 10] + [2, 1, 10, 10] | [5200.0, 10] + [2, 1, 11, 11] | [5700.0, 10] + ..... + [18, 7, null, null] | [147800.0, 40] + ... + [7, 2, null, null] | [28900.0, 20] + ... + [null, null, null, 99] | [49700.0, 10] + ..... + [null, null, null, null] | [2495000.0, 1000] + */ + assertEquals(635, starTreeDocuments.size()); + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + if (starTreeDocument.dimensions[0] != null + && starTreeDocument.dimensions[1] != null + && starTreeDocument.dimensions[2] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[1] != null + && starTreeDocument.dimensions[2] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null + && starTreeDocument.dimensions[2] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null + && starTreeDocument.dimensions[1] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[1] != null) { + assertTrue((long) starTreeDocument.metrics[1] == 20L || (long) starTreeDocument.metrics[1] == 40L); + } else if (starTreeDocument.dimensions[1] != null && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[1] != null) { + assertEquals(100L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null) { + assertEquals(40L, starTreeDocument.metrics[1]); + } + } + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 100, + 3, + 23199 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { + List dimList = List.of(0L, 1L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(5L, 6L, 8L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(5L, 6L, 7L, 8L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeField(MetricStat.VALUE_COUNT); + StarTreeValues starTreeValues = getStarTreeValues( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + compositeField, + "4" + ); + writeState = getWriteState(4, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + * Asserting following dim / metrics [ dim1, dim2 / Count [ metric] ] + [0, 0] | [0] + [1, 1] | [1] + [3, 3] | [3] + [4, 4] | [4] + [5, 5] | [10] + [6, 6] | [6] + [8, 8] | [8] + [null, 2] | [2] + [null, 7] | [7] + */ + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + assertEquals(9, builder.getStarTreeDocuments().size()); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + int count = 0; + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 9) { + if (Objects.equals(starTreeDocument.dimensions[0], 5L)) { + assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[0]); + } else { + assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); + } + } + } + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 9, 1000, 330); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOException { + List dimList1 = new ArrayList<>(500); + List docsWithField1 = new ArrayList<>(500); + + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 20; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 20 + j); + } + } + for (int i = 80; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 5 + j); + } + } + List dimList3 = new ArrayList<>(500); + List docsWithField3 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList3.add((long) i); + docsWithField3.add(i * 5 + j); + } + } + List dimList2 = new ArrayList<>(500); + List docsWithField2 = new ArrayList<>(500); + for (int i = 0; i < 500; i++) { + dimList2.add((long) 1); + docsWithField2.add(i); + } + + List dimList4 = new ArrayList<>(500); + List docsWithField4 = new ArrayList<>(500); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList4.add((long) i); + docsWithField4.add(i * 5 + j); + } + } + + List metricsList = new ArrayList<>(100); + List metricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(getLongFromDouble(i * 10.0)); + metricsWithField.add(i); + } + + List docCountMetricsList = new ArrayList<>(100); + List docCountMetricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(getLongFromDouble(i * 2)); + metricsWithField.add(i); + } + + compositeField = getStarTreeFieldWithDocCount(3, true); + StarTreeValues starTreeValues = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + docCountMetricsList, + docCountMetricsWithField, + compositeField + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + docCountMetricsList, + docCountMetricsWithField, + compositeField + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(401, starTreeDocuments.size()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 100, + compositeField.getStarTreeConfig().maxLeafDocs(), + 15345 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOException { + List dimList1 = new ArrayList<>(500); + List docsWithField1 = new ArrayList<>(500); + Map> expectedDimToValueMap = new HashMap<>(); + Map dimValueMap = new HashMap<>(); + for (int i = 0; i < 20; i++) { + for (int j = 0; j < 20; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 20 + j); + } + // metric = no of docs * 10.0 + dimValueMap.put((long) i, 200.0); + } + for (int i = 80; i < 100; i++) { + for (int j = 0; j < 5; j++) { + dimList1.add((long) i); + docsWithField1.add(i * 5 + j); + } + // metric = no of docs * 10.0 + dimValueMap.put((long) i, 50.0); + } + dimValueMap.put(Long.MAX_VALUE, 5000.0); + expectedDimToValueMap.put(0, dimValueMap); + dimValueMap = new HashMap<>(); + List dimList3 = new ArrayList<>(500); + List docsWithField3 = new ArrayList<>(500); + for (int i = 0; i < 500; i++) { + dimList3.add((long) 1); + docsWithField3.add(i); + dimValueMap.put((long) i, 10.0); + } + dimValueMap.put(Long.MAX_VALUE, 5000.0); + expectedDimToValueMap.put(2, dimValueMap); + dimValueMap = new HashMap<>(); + List dimList2 = new ArrayList<>(500); + List docsWithField2 = new ArrayList<>(500); + for (int i = 0; i < 500; i++) { + dimList2.add((long) i); + docsWithField2.add(i); + dimValueMap.put((long) i, 10.0); + } + dimValueMap.put(Long.MAX_VALUE, 200.0); + expectedDimToValueMap.put(1, dimValueMap); + dimValueMap = new HashMap<>(); + List dimList4 = new ArrayList<>(500); + List docsWithField4 = new ArrayList<>(500); + for (int i = 0; i < 500; i++) { + dimList4.add((long) 1); + docsWithField4.add(i); + dimValueMap.put((long) i, 10.0); + } + dimValueMap.put(Long.MAX_VALUE, 5000.0); + expectedDimToValueMap.put(3, dimValueMap); + List metricsList = new ArrayList<>(100); + List metricsWithField = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(getLongFromDouble(10.0)); + metricsWithField.add(i); + } + List metricsList1 = new ArrayList<>(100); + List metricsWithField1 = new ArrayList<>(100); + for (int i = 0; i < 500; i++) { + metricsList.add(1L); + metricsWithField.add(i); + } + compositeField = getStarTreeFieldWithDocCount(10, true); + StarTreeValues starTreeValues = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + metricsList1, + metricsWithField1, + compositeField + ); + + StarTreeValues starTreeValues2 = getStarTreeValues( + dimList1, + docsWithField1, + dimList2, + docsWithField2, + dimList3, + docsWithField3, + dimList4, + docsWithField4, + metricsList, + metricsWithField, + metricsList1, + metricsWithField1, + compositeField + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); + List starTreeDocuments = builder.getStarTreeDocuments(); + Map> dimValueToDocIdMap = new HashMap<>(); + traverseStarTree(builder.rootNode, dimValueToDocIdMap, true); + for (Map.Entry> entry : dimValueToDocIdMap.entrySet()) { + int dimId = entry.getKey(); + if (dimId == -1) continue; + Map map = expectedDimToValueMap.get(dimId); + for (Map.Entry dimValueToDocIdEntry : entry.getValue().entrySet()) { + long dimValue = dimValueToDocIdEntry.getKey(); + int docId = dimValueToDocIdEntry.getValue(); + assertEquals(map.get(dimValue) * 2, starTreeDocuments.get(docId).metrics[0]); + } + } + assertEquals(1041, starTreeDocuments.size()); + validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 500, + compositeField.getStarTreeConfig().maxLeafDocs(), + 31779 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + public void testMergeFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L); + List docsWithField = List.of(0, 1, 2, 3, 4, 6); + List dimList2 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); + List dimList7 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField7 = List.of(0, 1, 2, 3, 4, 6); + + List dimList5 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List docsWithField5 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList1 = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField1 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField4 = List.of(0, 1, 3, 4); + List dimList8 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField8 = List.of(0, 1, 3, 4); + + List dimList6 = List.of(5L, 6L, 7L, 8L); + List docsWithField6 = List.of(0, 1, 2, 3); + List metricsList21 = List.of( + getLongFromDouble(50.0), + getLongFromDouble(60.0), + getLongFromDouble(70.0), + getLongFromDouble(80.0), + getLongFromDouble(90.0) + ); + List metricsWithField21 = List.of(0, 1, 2, 3, 4); + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeFieldWithDateDimension(); + StarTreeValues starTreeValues = getStarTreeValuesWithDates( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(dimList7, docsWithField7), + getSortedNumericMock(dimList5, docsWithField5), + getSortedNumericMock(metricsList, metricsWithField), + getSortedNumericMock(metricsList1, metricsWithField1), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValuesWithDates( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(dimList8, docsWithField8), + getSortedNumericMock(dimList6, docsWithField6), + getSortedNumericMock(metricsList2, metricsWithField2), + getSortedNumericMock(metricsList21, metricsWithField21), + compositeField, + "4" + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [1655287972000, 1655287972000, 1655287972000, 3] | [30.0, 3] + [1655288032000, 1655288032000, 1655288032000, 2] | [20.0, 2] + [1655288032000, 1655288032000, 1655288032000, 8] | [80.0, 8] + [1655288092000, 1655288092000, 1655288092000, 1] | [10.0, 1] + [1655288092000, 1655288092000, 1655288092000, 4] | [40.0, 4] + [1655288092000, 1655288092000, 1655288092000, 6] | [60.0, 6] + [1655288152000, 1655288152000, 1655288152000, 0] | [0.0, 0] + [1655288152000, 1655288152000, 1655288152000, 5] | [50.0, 5] + [null, null, null, 5] | [50.0, 5] + [null, null, null, 7] | [70.0, 7] + */ + int count = 0; + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[1], 0); + assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[0]); + } + assertEquals(10, count); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 10, + compositeField.getStarTreeConfig().maxLeafDocs(), + 231 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + + private StarTreeValues getStarTreeValuesWithDates( + SortedNumericDocValues dimList, + SortedNumericDocValues dimList2, + SortedNumericDocValues dimList4, + SortedNumericDocValues dimList3, + SortedNumericDocValues metricsList, + SortedNumericDocValues metricsList1, + StarTreeField sf, + String number + ) { + Map> dimDocIdSetIterators = Map.of( + "field1_minute", + () -> new SortedNumericStarTreeValuesIterator(dimList), + "field1_half-hour", + () -> new SortedNumericStarTreeValuesIterator(dimList4), + "field1_hour", + () -> new SortedNumericStarTreeValuesIterator(dimList2), + "field3", + () -> new SortedNumericStarTreeValuesIterator(dimList3) + ); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList) + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(1).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList1) + ); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number), null); + } + + private StarTreeValues getStarTreeValues( + SortedNumericDocValues dimList, + SortedNumericDocValues dimList2, + SortedNumericDocValues metricsList, + StarTreeField sf, + String number + ) { + SortedNumericDocValues d1sndv = dimList; + SortedNumericDocValues d2sndv = dimList2; + SortedNumericDocValues m1sndv = metricsList; + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv) + ); + + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + for (Metric metric : sf.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); + } + } + + StarTreeValues starTreeValues = new StarTreeValues( + sf, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + Map.of(SEGMENT_DOCS_COUNT, number), + null + ); + return starTreeValues; + } + + private StarTreeValues getStarTreeValues( + List dimList1, + List docsWithField1, + List dimList2, + List docsWithField2, + List dimList3, + List docsWithField3, + List dimList4, + List docsWithField4, + List metricsList, + List metricsWithField, + List metricsList1, + List metricsWithField1, + StarTreeField sf + ) { + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList1, docsWithField1); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues d3sndv = getSortedNumericMock(dimList3, docsWithField3); + SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv), + "field5", + () -> new SortedNumericStarTreeValuesIterator(d3sndv), + "field8", + () -> new SortedNumericStarTreeValuesIterator(d4sndv) + ); + + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(m1sndv) + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "_doc_count", + sf.getMetrics().get(1).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(m2sndv) + ); + // metricDocIdSetIterators.put("field2", () -> m1sndv); + // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); + StarTreeValues starTreeValues = new StarTreeValues( + sf, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + getAttributes(500), + null + ); + return starTreeValues; + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderSortAndAggregateTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderSortAndAggregateTests.java new file mode 100644 index 0000000000000..851535bcc2b26 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderSortAndAggregateTests.java @@ -0,0 +1,695 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getDimensionIterators; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getMetricIterators; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; + +public class StarTreeBuilderSortAndAggregateTests extends StarTreeBuilderTestCase { + + public StarTreeBuilderSortAndAggregateTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + super(buildMode); + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP }, + new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP } + ); + } + + public void test_sortAndAggregateStarTreeDocuments() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0, 10L } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, 10L } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0, 10L } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, null } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, null } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = (Long) starTreeDocuments[i].metrics[5]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 11L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 21L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + } + + public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { Double.MAX_VALUE, 10.0, randomDouble(), 8.0, 20.0, 100L } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0, 100L } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 14.0, Double.MIN_VALUE, randomDouble(), 6.0, 24.0, 100L } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0, 100L } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0, 100L } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { Double.MAX_VALUE + 9, 14.0, 2L, 8.0, 20.0, 200L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, Double.MIN_VALUE + 22, 3L, 6.0, 24.0, 300L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = (Long) starTreeDocuments[i].metrics[5]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); + + } + + public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, + new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, + new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, + new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, + new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, + new Object[] { 11.0, 16.0, randomDouble(), 8.0, 13.0 } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Long[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + } + + public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 12.0, 10.0, randomDouble(), 8.0, 20.0 }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 10.0, 6.0, randomDouble(), 12.0, 10.0 }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 14.0, 12.0, randomDouble(), 6.0, 24.0 }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 9.0, 4.0, randomDouble(), 9.0, 12.0 }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 11.0, null, randomDouble(), 8.0, 13.0 }); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 18.0, 3L, 6.0, 24.0, 3L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + } + + public void test_sortAndAggregateStarTreeDocuments_nullMetricField() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + // Setting second metric iterator as empty sorted numeric , indicating a metric field is null + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, null } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, null } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, 4L, 3L, 4L }, + new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { 3L, 4L, 2L, 1L }, + new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, null } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 0.0, 2L, 8.0, 20.0, 11L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 0.0, 3L, 6.0, 24.0, 3L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = starTreeDocuments[i].metrics[5] != null ? (Long) starTreeDocuments[i].metrics[5] : null; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + } + + public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionField() throws IOException { + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + // Setting second metric iterator as empty sorted numeric , indicating a metric field is null + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { 2L, null, 3L, 4L }, + new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, null } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { null, 4L, 2L, 1L }, + new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, null } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, 4L, 2L, 1L }, + new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { 2L, null, 3L, 4L }, + new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { -1L, 4L, 2L, 1L }, + new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, null } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 11.0, 0.0, 1L, 8.0, 13.0, 1L }), + new StarTreeDocument(new Long[] { 2L, null, 3L, 4L }, new Object[] { 21.0, 0.0, 2L, 8.0, 20.0, 11L }), + new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 24.0, 0.0, 2L, 6.0, 24.0, 2L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = starTreeDocuments[i].metrics[5] != null ? (long) starTreeDocuments[i].metrics[5] : null; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + + assertFalse(expectedStarTreeDocumentIterator.hasNext()); + + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + } + + public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics() throws IOException { + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + // Setting second metric iterator as empty sorted numeric , indicating a metric field is null + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, null, null, null } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, null, null, null } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, null, null, null } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, null, null, null } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, null, null, null } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { 0.0, 0.0, 0L, null, null, 5L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + Long metric1 = starTreeDocuments[i].metrics[0] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]) + : null; + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + Long metric3 = starTreeDocuments[i].metrics[2] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]) + : null; + Long metric4 = starTreeDocuments[i].metrics[3] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]) + : null; + Long metric5 = starTreeDocuments[i].metrics[4] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]) + : null; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + } + + public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetrics() throws IOException { + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + double sumValue = randomDouble(); + double minValue = randomDouble(); + double maxValue = randomDouble(); + + // Setting second metric iterator as empty sorted numeric , indicating a metric field is null + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, randomDouble(), null, maxValue } + ); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { null, null, null, null, null }); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { null, null, null, minValue, null } + ); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { null, null, null, null, null }); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { sumValue, null, randomDouble(), null, null } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { sumValue, 0.0, 2L, minValue, maxValue, 5L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + Long metric1 = starTreeDocuments[i].metrics[0] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]) + : null; + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + Long metric3 = starTreeDocuments[i].metrics[2] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]) + : null; + Long metric4 = starTreeDocuments[i].metrics[3] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]) + : null; + Long metric5 = starTreeDocuments[i].metrics[4] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]) + : null; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, null } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + } + + public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + // Setting second metric iterator as empty sorted numeric , indicating a metric field is null + starTreeDocuments[0] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { 12.0, null, randomDouble(), 8.0, 20.0, 10L } + ); + starTreeDocuments[1] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { 10.0, null, randomDouble(), 12.0, 10.0, 10L } + ); + starTreeDocuments[2] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { 14.0, null, randomDouble(), 6.0, 24.0, 10L } + ); + starTreeDocuments[3] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { 9.0, null, randomDouble(), 9.0, 12.0, 10L } + ); + starTreeDocuments[4] = new StarTreeDocument( + new Long[] { null, null, null, null }, + new Object[] { 11.0, null, randomDouble(), 8.0, 13.0, 10L } + ); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { null, null, null, null }, new Object[] { 56.0, 0.0, 5L, 6.0, 24.0, 50L }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + Long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + Long metric4 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[3]); + Long metric5 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[4]); + Long metric6 = (Long) starTreeDocuments[i].metrics[5]; + segmentStarTreeDocuments[i] = new StarTreeDocument( + starTreeDocuments[i].dimensions, + new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } + ); + } + SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimsIterators, + metricsIterators + ); + + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); + } + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java new file mode 100644 index 0000000000000..4c854f7546197 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -0,0 +1,368 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.compositeindex.CompositeIndexConstants; +import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; +import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public abstract class StarTreeBuilderTestCase extends OpenSearchTestCase { + private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; + protected MapperService mapperService; + protected List fields = List.of(); + protected List metrics; + protected Directory directory; + protected FieldInfo[] fieldsInfo; + protected StarTreeField compositeField; + protected Map fieldProducerMap; + protected SegmentWriteState writeState; + protected BaseStarTreeBuilder builder; + protected IndexOutput dataOut; + protected IndexOutput metaOut; + protected DocValuesConsumer docValuesConsumer; + protected String dataFileName; + protected String metaFileName; + protected List dimensionsOrder; + + public StarTreeBuilderTestCase(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + this.buildMode = buildMode; + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP }, + new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP } + ); + } + + @Before + public void setup() throws IOException { + fields = List.of("field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", "field9", "field10"); + + dimensionsOrder = List.of( + new NumericDimension("field1"), + new NumericDimension("field3"), + new NumericDimension("field5"), + new NumericDimension("field8") + ); + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.VALUE_COUNT)), + new Metric("field9", List.of(MetricStat.MIN)), + new Metric("field10", List.of(MetricStat.MAX)), + new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)) + ); + + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), getBuildMode()) + ); + directory = newFSDirectory(createTempDir()); + + fieldsInfo = new FieldInfo[fields.size()]; + fieldProducerMap = new HashMap<>(); + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + fieldProducerMap.put(fields.get(i), docValuesProducer); + } + writeState = getWriteState(5, UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8)); + + dataFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.DATA_EXTENSION + ); + dataOut = writeState.directory.createOutput(dataFileName, writeState.context); + + metaFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite99DocValuesFormat.META_EXTENSION + ); + metaOut = writeState.directory.createOutput(metaFileName, writeState.context); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper4 = new NumberFieldMapper.Builder("field9", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper5 = new NumberFieldMapper.Builder("field10", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3, numberFieldMapper4, numberFieldMapper5), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + docValuesConsumer = mock(DocValuesConsumer.class); + } + + protected BaseStarTreeBuilder getStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) throws IOException { + switch (buildMode) { + case ON_HEAP: + return new OnHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); + case OFF_HEAP: + return new OffHeapStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); + default: + throw new IllegalArgumentException("Invalid build mode: " + buildMode); + } + } + + protected StarTreeFieldConfiguration.StarTreeBuildMode getBuildMode() { + return buildMode; + } + + protected void validateStarTreeFileFormats( + InMemoryTreeNode rootNode, + int numDocs, + StarTreeMetadata expectedStarTreeMetadata, + List expectedStarTreeDocuments + ) throws IOException { + BuilderTestsUtils.validateStarTreeFileFormats( + rootNode, + numDocs, + expectedStarTreeMetadata, + expectedStarTreeDocuments, + dataFileName, + metaFileName, + builder, + compositeField, + writeState, + directory + ); + + } + + SegmentWriteState getWriteState(int numDocs, byte[] id) { + return BuilderTestsUtils.getWriteState(numDocs, id, fieldsInfo, directory); + } + + SegmentReadState getReadState(int numDocs, List dimensionFields, List metrics) { + return BuilderTestsUtils.getReadState(numDocs, dimensionFields, metrics, compositeField, writeState, directory); + } + + protected Map getAttributes(int numSegmentDocs) { + return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); + } + + protected List getStarTreeDimensionNames(List dimensionsOrder) { + List dimensionNames = new ArrayList<>(); + for (Dimension dimension : dimensionsOrder) { + dimensionNames.addAll(dimension.getSubDimensionNames()); + } + return dimensionNames; + } + + protected StarTreeField getStarTreeField(MetricStat count) { + Dimension d1 = new NumericDimension("field1"); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(count)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + + protected StarTreeField getStarTreeFieldWithDocCount(int maxLeafDocs, boolean includeDocCountMetric) { + Dimension d1 = new NumericDimension("field1"); + Dimension d2 = new NumericDimension("field3"); + Dimension d3 = new NumericDimension("field5"); + Dimension d4 = new NumericDimension("field8"); + List dims = List.of(d1, d2, d3, d4); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = null; + if (includeDocCountMetric) { + m2 = new Metric("_doc_count", List.of(MetricStat.DOC_COUNT)); + } + List metrics = m2 == null ? List.of(m1) : List.of(m1, m2); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(maxLeafDocs, new HashSet<>(), getBuildMode()); + StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + return sf; + } + + protected void assertStarTreeDocuments( + List resultStarTreeDocuments, + Iterator expectedStarTreeDocumentIterator + ) { + Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); + while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); + assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); + assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); + } + } + + protected static List getExpectedStarTreeDocumentIterator() { + return List.of( + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), + new StarTreeDocument(new Long[] { null, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), + new StarTreeDocument(new Long[] { null, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), + new StarTreeDocument(new Long[] { null, 4L, null, 1L }, new Object[] { 35.0, 34.0, 3L, 6.0, 24.0, 3L }), + new StarTreeDocument(new Long[] { null, 4L, null, 4L }, new Object[] { 21.0, 14.0, 2L, 8.0, 20.0, 2L }), + new StarTreeDocument(new Long[] { null, 4L, null, null }, new Object[] { 56.0, 48.0, 5L, 6.0, 24.0, 5L }) + ); + } + + protected long getLongFromDouble(double value) { + return NumericUtils.doubleToSortableLong(value); + } + + protected StarTreeMetadata getStarTreeMetadata(List fields, int segmentAggregatedDocCount, int maxLeafDocs, int dataLength) { + return new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + fields, + compositeField.getMetrics(), + segmentAggregatedDocCount, + builder.numStarTreeDocs, + maxLeafDocs, + Set.of(), + getBuildMode(), + 0, + dataLength + ); + } + + protected StarTreeField getStarTreeFieldWithDateDimension() { + List intervals = new ArrayList<>(); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + intervals.add(DataCubeDateTimeUnit.HALF_HOUR_OF_DAY); + Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.VALUE_COUNT, MetricStat.SUM)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(10, new HashSet<>(), getBuildMode()); + StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + return sf; + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + if (builder != null) { + builder.close(); + } + docValuesConsumer.close(); + metaOut.close(); + dataOut.close(); + directory.close(); + } +} From 5279d21f8c85e78a91159af6e17e52789355f2a4 Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:59:54 -0700 Subject: [PATCH 198/357] Change successfulSearchShardIndices to Set (#16110) * Change successfulSearchShardIndices to Set Signed-off-by: David Zane * Update CHANGELOG.md Signed-off-by: Ankit Jain --------- Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 2 +- .../action/search/AbstractSearchAsyncAction.java | 4 +++- .../opensearch/action/search/SearchRequestContext.java | 9 +++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ac7740eea6e..d8732d1ffa5cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) -- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) +- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110)) - [Tiered Caching] Segmented cache changes ([#16047](https://github.com/opensearch-project/OpenSearch/pull/16047)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 836083f91b995..85ea34e442c8f 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -755,7 +755,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits()); searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length); searchRequestContext.setSuccessfulSearchShardIndices( - results.getSuccessfulResults().map(result -> result.getSearchShardTarget().getIndex()).collect(Collectors.toSet()) + results.getSuccessfulResults() + .map(result -> result.getSearchShardTarget().getShardId().getIndex()) + .collect(Collectors.toSet()) ); onPhaseEnd(searchRequestContext); onRequestEnd(searchRequestContext); diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java index 376cf71448d5c..398896734280e 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.search.TotalHits; import org.opensearch.common.annotation.InternalApi; +import org.opensearch.core.index.Index; import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import java.util.ArrayList; @@ -37,7 +38,7 @@ public class SearchRequestContext { private final Map phaseTookMap; private TotalHits totalHits; private final EnumMap shardStats; - private Set successfulSearchShardIndices; + private Set successfulSearchShardIndices; private final SearchRequest searchRequest; private final LinkedBlockingQueue phaseResourceUsage; @@ -144,15 +145,15 @@ public SearchRequest getRequest() { return searchRequest; } - void setSuccessfulSearchShardIndices(Set successfulSearchShardIndices) { + void setSuccessfulSearchShardIndices(Set successfulSearchShardIndices) { this.successfulSearchShardIndices = successfulSearchShardIndices; } /** - * @return A {@link List} of {@link String} representing the names of the indices that were + * @return A {@link Set} of {@link Index} representing the names of the indices that were * successfully queried at the shard level. */ - public Set getSuccessfulSearchShardIndices() { + public Set getSuccessfulSearchShardIndices() { return successfulSearchShardIndices; } } From e8e041b00f26767e6037d852f2022241f181b08a Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:05:38 +0530 Subject: [PATCH 199/357] Add more concurrent Snapshot V2 Integ Tests (#16207) Signed-off-by: Gaurav Bafna --- .../snapshots/ConcurrentSnapshotsV2IT.java | 148 +++++++++++++++++- 1 file changed, 146 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index 4ecbe221550e6..78497cac41d46 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -9,6 +9,7 @@ package org.opensearch.snapshots; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; @@ -16,6 +17,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.rest.RestStatus; import org.opensearch.remotestore.RemoteSnapshotIT; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -479,7 +481,6 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception { assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo())); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191") public void testDeleteWhileV2CreateOngoing() throws Exception { final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); @@ -538,7 +539,7 @@ public void testDeleteWhileV2CreateOngoing() throws Exception { assertThat(snapInfo, contains(csr.getSnapshotInfo())); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191") + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205") public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); internalCluster().startDataOnlyNode(pinnedTimestampSettings()); @@ -569,6 +570,7 @@ public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { indexDocuments(client, indexName1, numDocsInIndex1); indexDocuments(client, indexName2, numDocsInIndex2); ensureGreen(indexName1, indexName2); + ensureGreen(); startFullSnapshot(repoName, "snapshot-v1").actionGet(); startFullSnapshot(repoName, "snapshot-v1-2").actionGet(); @@ -597,6 +599,148 @@ public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception { assertTrue(startCloneSnapshot.actionGet().isAcknowledged()); List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); assertEquals(3, snapInfo.size()); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone") + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern("(.+)") + .setRenameReplacement("$1-copy") + .get(); + assertEquals(restoreSnapshotResponse.status(), RestStatus.OK); + ensureGreen(); + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205") + public void testDeleteAndCloneV1WhileCreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(10, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + startFullSnapshot(repoName, "snapshot-v1-2").actionGet(); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture startDeleteSnapshot = startDeleteSnapshot(repoName, "snapshot-v1"); + ActionFuture startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1-2", "snapshot-v1-2-clone"); + + unblockNode(repoName, clusterManagerName); + snapshotFuture.actionGet(); + assertTrue(startDeleteSnapshot.actionGet().isAcknowledged()); + assertTrue(startCloneSnapshot.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(3, snapInfo.size()); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone") + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern("(.+)") + .setRenameReplacement("$1-copy") + .get(); + assertEquals(restoreSnapshotResponse.status(), RestStatus.OK); + ensureGreen(); + } + + public void testCloneV1WhileV2CreateOngoing() throws Exception { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + internalCluster().startDataOnlyNode(pinnedTimestampSettings()); + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String repoName = "test-create-snapshot-repo"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Settings.Builder settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false); + createRepository(repoName, "mock", settings); + + Client client = client(); + Settings indexSettings = getIndexSettings(20, 0).build(); + createIndex(indexName1, indexSettings); + + Settings indexSettings2 = getIndexSettings(15, 0).build(); + createIndex(indexName2, indexSettings2); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexDocuments(client, indexName1, numDocsInIndex1); + indexDocuments(client, indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + startFullSnapshot(repoName, "snapshot-v1").actionGet(); + + // Creating a v2 repo + settings = Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true); + createRepository(repoName, "mock", settings); + + blockClusterManagerOnWriteIndexFile(repoName); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-v2"); + awaitNumberOfSnapshotsInProgress(1); + + ActionFuture startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1", "snapshot-v1-2-clone"); + + unblockNode(repoName, clusterManagerName); + CreateSnapshotResponse csr = snapshotFuture.actionGet(); + assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); + assertTrue(startCloneSnapshot.actionGet().isAcknowledged()); + List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); + assertEquals(3, snapInfo.size()); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone") + .setWaitForCompletion(true) + .setIndices(indexName1) + .setRenamePattern(indexName1) + .setRenamePattern("(.+)") + .setRenameReplacement("$1-copy") + .get(); + assertEquals(restoreSnapshotResponse.status(), RestStatus.OK); + ensureGreen(); } protected ActionFuture startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) { From aad325f5d3c09866b63ead0c07f9c071d15c483a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:06:32 +0800 Subject: [PATCH 200/357] Bump com.azure:azure-json from 1.1.0 to 1.3.0 in /plugins/repository-azure (#16217) * Bump com.azure:azure-json in /plugins/repository-azure Bumps [com.azure:azure-json](https://github.com/Azure/azure-sdk-for-java) from 1.1.0 to 1.3.0. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/v1.1.0...v1.3.0) --- updated-dependencies: - dependency-name: com.azure:azure-json dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/azure-json-1.1.0.jar.sha1 | 1 - plugins/repository-azure/licenses/azure-json-1.3.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-json-1.1.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-json-1.3.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d8732d1ffa5cc..0cad8c764a8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) +- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 4baf79e619be9..e2d1c15cdb36f 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -45,7 +45,7 @@ opensearchplugin { dependencies { api 'com.azure:azure-core:1.51.0' - api 'com.azure:azure-json:1.1.0' + api 'com.azure:azure-json:1.3.0' api 'com.azure:azure-xml:1.1.0' api 'com.azure:azure-storage-common:12.25.1' api 'com.azure:azure-core-http-netty:1.15.4' diff --git a/plugins/repository-azure/licenses/azure-json-1.1.0.jar.sha1 b/plugins/repository-azure/licenses/azure-json-1.1.0.jar.sha1 deleted file mode 100644 index e44ee47c40253..0000000000000 --- a/plugins/repository-azure/licenses/azure-json-1.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f21cea72f54a6af3b0bb6831eb3874bd4afd213 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-json-1.3.0.jar.sha1 b/plugins/repository-azure/licenses/azure-json-1.3.0.jar.sha1 new file mode 100644 index 0000000000000..8562474bd1472 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-json-1.3.0.jar.sha1 @@ -0,0 +1 @@ +11b6a0708e9d6c90a1a76574c7720edce47dacc1 \ No newline at end of file From 96082f7fd263dbd9a125c1c99e81fd6ed51b31f6 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 8 Oct 2024 15:15:34 +0530 Subject: [PATCH 201/357] Close RemoteStorePinnedTimestampService on Node.close() (#16228) Signed-off-by: Sachin Kale --- server/src/main/java/org/opensearch/node/Node.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index ddba10369ff05..5c965b06a4b69 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1868,6 +1868,7 @@ public synchronized void close() throws IOException { toClose.add(() -> stopWatch.stop().start("transport")); toClose.add(injector.getInstance(TransportService.class)); toClose.add(nodeService.getTaskCancellationMonitoringService()); + toClose.add(injector.getInstance(RemoteStorePinnedTimestampService.class)); for (LifecycleComponent plugin : pluginLifecycleComponents) { toClose.add(() -> stopWatch.stop().start("plugin(" + plugin.getClass().getName() + ")")); @@ -1898,6 +1899,7 @@ public synchronized void close() throws IOException { if (logger.isTraceEnabled()) { toClose.add(() -> logger.trace("Close times for each service:\n{}", stopWatch.prettyPrint())); } + IOUtils.close(toClose); logger.info("closed"); } From febbc00132a13e04e65e4ef1996cc73eda4729de Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 8 Oct 2024 13:44:04 -0400 Subject: [PATCH 202/357] Add TLSv1.3 in jdk.tls.client.protocols systemProp in gradle.properties (#16237) * Remove force to TLSv1.2 in gradle.properties Signed-off-by: Craig Perkins * Add TLSv1.3 Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- gradle.properties | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index 4e8c5b98116c1..d4634f1a7aeea 100644 --- a/gradle.properties +++ b/gradle.properties @@ -31,9 +31,7 @@ systemProp.org.gradle.dependency.duplicate.project.detection=false # Enforce the build to fail on deprecated gradle api usage systemProp.org.gradle.warning.mode=fail -# forcing to use TLS1.2 to avoid failure in vault -# see https://github.com/hashicorp/vault/issues/8750#issuecomment-631236121 -systemProp.jdk.tls.client.protocols=TLSv1.2 +systemProp.jdk.tls.client.protocols=TLSv1.2,TLSv1.3 # jvm args for faster test execution by default systemProp.tests.jvm.argline=-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m From 2ed8ff002e3fe1e45c7d0a3f26e67e4352adc8f5 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:11:27 -0700 Subject: [PATCH 203/357] Attempt to delete backport branches only for OpenSearch repo (#16240) Signed-off-by: Sayali Gaikawad --- .github/workflows/delete_backport_branch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 1c73d3d250ce3..22ce83c69a5d8 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - if: startsWith(github.event.pull_request.head.ref,'backport/') + if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/') steps: - name: Delete merged branch uses: actions/github-script@v7 From 62b153750dd13cd59f62ffbb76cbe35527b6fafa Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:00:18 -0700 Subject: [PATCH 204/357] Fix Cache backward compatibility breaking change (#16238) * Fix Cache breaking change Signed-off-by: Sagar Upadhyaya * Empty commit to trigger build Signed-off-by: Sagar Upadhyaya --------- Signed-off-by: Sagar Upadhyaya --- server/src/main/java/org/opensearch/common/cache/Cache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/Cache.java b/server/src/main/java/org/opensearch/common/cache/Cache.java index da683bfff088d..e01a1223955ed 100644 --- a/server/src/main/java/org/opensearch/common/cache/Cache.java +++ b/server/src/main/java/org/opensearch/common/cache/Cache.java @@ -116,13 +116,13 @@ public class Cache { private RemovalListener removalListener = notification -> {}; private final int numberOfSegments; - private static final int NUMBER_OF_SEGMENTS_DEFAULT = 256; + public static final int NUMBER_OF_SEGMENTS = 256; Cache(final int numberOfSegments) { if (numberOfSegments != -1) { this.numberOfSegments = numberOfSegments; } else { - this.numberOfSegments = NUMBER_OF_SEGMENTS_DEFAULT; + this.numberOfSegments = NUMBER_OF_SEGMENTS; } this.segments = new CacheSegment[this.numberOfSegments]; for (int i = 0; i < this.numberOfSegments; i++) { From b5917c5c349c4c931fd278f59f5fadfbcf7f0379 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 9 Oct 2024 05:52:02 +0530 Subject: [PATCH 205/357] [Star tree] Moving compositeCodec to composite912Codec and dropping composite99Codec (#16227) * Refactoring builder tests * composite912 codec refactor changes --------- Signed-off-by: Bharathwaj G --- .../composite/CompositeCodecFactory.java | 3 +- .../LuceneDocValuesProducerFactory.java | 4 +- .../composite912/Composite912Codec.java | 3 +- .../Composite912DocValuesFormat.java} | 22 +-- .../Composite912DocValuesReader.java} | 34 ++--- .../Composite912DocValuesWriter.java} | 40 +++--- .../composite99/Composite99Codec.java | 51 ------- .../composite/composite99/package-info.java | 12 -- .../startree/index/StarTreeValues.java | 2 +- .../index/engine/SegmentsStats.java | 10 +- .../services/org.apache.lucene.codecs.Codec | 1 - .../LuceneDocValuesConsumerFactoryTests.java | 4 +- .../LuceneDocValuesProducerFactoryTests.java | 4 +- .../builder/BaseStarTreeBuilderTests.java | 6 +- .../startree/builder/BuilderTestsUtils.java | 14 +- .../builder/StarTreeBuildMetricTests.java | 42 +++--- .../StarTreeBuilderFlushFlowTests.java | 34 ++--- .../StarTreeBuilderMergeFlowTests.java | 125 +++++++++--------- .../builder/StarTreeBuilderTestCase.java | 6 +- 19 files changed, 176 insertions(+), 241 deletions(-) rename server/src/main/java/org/opensearch/index/codec/composite/{composite99/Composite99DocValuesFormat.java => composite912/Composite912DocValuesFormat.java} (79%) rename server/src/main/java/org/opensearch/index/codec/composite/{composite99/Composite99DocValuesReader.java => composite912/Composite912DocValuesReader.java} (91%) rename server/src/main/java/org/opensearch/index/codec/composite/{composite99/Composite99DocValuesWriter.java => composite912/Composite912DocValuesWriter.java} (91%) delete mode 100644 server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java delete mode 100644 server/src/main/java/org/opensearch/index/codec/composite/composite99/package-info.java diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java index f5003417c1298..674773bb86354 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java @@ -13,7 +13,6 @@ import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.composite912.Composite912Codec; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; import org.opensearch.index.mapper.MapperService; import java.util.HashMap; @@ -33,7 +32,7 @@ public class CompositeCodecFactory { // we can use this to track the latest composite codec - public static final String COMPOSITE_CODEC = Composite99Codec.COMPOSITE_INDEX_CODEC_NAME; + public static final String COMPOSITE_CODEC = Composite912Codec.COMPOSITE_INDEX_CODEC_NAME; public CompositeCodecFactory() {} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java index d85205d239648..80555597fd48a 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper; import org.apache.lucene.index.SegmentReadState; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import java.io.IOException; @@ -39,7 +39,7 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec( ) throws IOException { switch (compositeCodec) { - case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME: + case Composite912Codec.COMPOSITE_INDEX_CODEC_NAME: Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper( state, dataCodec, diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java index 9bbff7d344c2f..6f7a3c6852cf6 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java @@ -15,7 +15,6 @@ import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; import org.opensearch.index.mapper.MapperService; /** @@ -53,6 +52,6 @@ protected Composite912Codec(String name, Codec delegate, MapperService mapperSer @Override public DocValuesFormat docValuesFormat() { - return new Composite99DocValuesFormat(mapperService); + return new Composite912DocValuesFormat(mapperService); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesFormat.java similarity index 79% rename from server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java rename to server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesFormat.java index 63fe5a6af76f7..3a8e748e5a9e6 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite.composite99; +package org.opensearch.index.codec.composite.composite912; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; @@ -25,7 +25,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite99DocValuesFormat extends DocValuesFormat { +public class Composite912DocValuesFormat extends DocValuesFormat { /** * Creates a new docvalues format. * @@ -38,10 +38,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat { private final MapperService mapperService; /** Data codec name for Composite Doc Values Format */ - public static final String DATA_CODEC_NAME = "Composite99FormatData"; + public static final String DATA_CODEC_NAME = "Composite912FormatData"; /** Meta codec name for Composite Doc Values Format */ - public static final String META_CODEC_NAME = "Composite99FormatMeta"; + public static final String META_CODEC_NAME = "Composite912FormatMeta"; /** Filename extension for the composite index data */ public static final String DATA_EXTENSION = "cid"; @@ -50,10 +50,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat { public static final String META_EXTENSION = "cim"; /** Data doc values codec name for Composite Doc Values Format */ - public static final String DATA_DOC_VALUES_CODEC = "Composite99DocValuesData"; + public static final String DATA_DOC_VALUES_CODEC = "Composite912DocValuesData"; /** Meta doc values codec name for Composite Doc Values Format */ - public static final String META_DOC_VALUES_CODEC = "Composite99DocValuesMetadata"; + public static final String META_DOC_VALUES_CODEC = "Composite912DocValuesMetadata"; /** Filename extension for the composite index data doc values */ public static final String DATA_DOC_VALUES_EXTENSION = "cidvd"; @@ -68,15 +68,15 @@ public class Composite99DocValuesFormat extends DocValuesFormat { public static final int VERSION_CURRENT = VERSION_START; // needed for SPI - public Composite99DocValuesFormat() { + public Composite912DocValuesFormat() { this(new Lucene90DocValuesFormat(), null); } - public Composite99DocValuesFormat(MapperService mapperService) { + public Composite912DocValuesFormat(MapperService mapperService) { this(new Lucene90DocValuesFormat(), mapperService); } - public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) { + public Composite912DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) { super(delegate.getName()); this.delegate = delegate; this.mapperService = mapperService; @@ -84,11 +84,11 @@ public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapper @Override public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new Composite99DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService); + return new Composite912DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService); } @Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Composite99DocValuesReader(delegate.fieldsProducer(state), state); + return new Composite912DocValuesReader(delegate.fieldsProducer(state), state); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java similarity index 91% rename from server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java rename to server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java index 7901336151c8e..bb8a07d856d87 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite.composite99; +package org.opensearch.index.codec.composite.composite912; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -56,8 +56,8 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader { - private static final Logger logger = LogManager.getLogger(Composite99DocValuesReader.class); +public class Composite912DocValuesReader extends DocValuesProducer implements CompositeIndexReader { + private static final Logger logger = LogManager.getLogger(Composite912DocValuesReader.class); private final DocValuesProducer delegate; private IndexInput dataIn; @@ -69,20 +69,20 @@ public class Composite99DocValuesReader extends DocValuesProducer implements Com private final List compositeFieldInfos = new ArrayList<>(); private SegmentReadState readState; - public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException { + public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException { this.delegate = producer; this.fields = new ArrayList<>(); String metaFileName = IndexFileNames.segmentFileName( readState.segmentInfo.name, readState.segmentSuffix, - Composite99DocValuesFormat.META_EXTENSION + Composite912DocValuesFormat.META_EXTENSION ); String dataFileName = IndexFileNames.segmentFileName( readState.segmentInfo.name, readState.segmentSuffix, - Composite99DocValuesFormat.DATA_EXTENSION + Composite912DocValuesFormat.DATA_EXTENSION ); boolean success = false; @@ -92,9 +92,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r dataIn = readState.directory.openInput(dataFileName, readState.context); CodecUtil.checkIndexHeader( dataIn, - Composite99DocValuesFormat.DATA_CODEC_NAME, - Composite99DocValuesFormat.VERSION_START, - Composite99DocValuesFormat.VERSION_CURRENT, + Composite912DocValuesFormat.DATA_CODEC_NAME, + Composite912DocValuesFormat.VERSION_START, + Composite912DocValuesFormat.VERSION_CURRENT, readState.segmentInfo.getId(), readState.segmentSuffix ); @@ -105,9 +105,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r try { CodecUtil.checkIndexHeader( metaIn, - Composite99DocValuesFormat.META_CODEC_NAME, - Composite99DocValuesFormat.VERSION_START, - Composite99DocValuesFormat.VERSION_CURRENT, + Composite912DocValuesFormat.META_CODEC_NAME, + Composite912DocValuesFormat.VERSION_START, + Composite912DocValuesFormat.VERSION_CURRENT, readState.segmentInfo.getId(), readState.segmentSuffix ); @@ -190,12 +190,12 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r // initialize star-tree doc values producer compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( - Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + Composite912Codec.COMPOSITE_INDEX_CODEC_NAME, this.readState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); } catch (Throwable t) { diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java similarity index 91% rename from server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java rename to server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java index 0d4e35f7c3ab8..2225870afae8e 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.composite.composite99; +package org.opensearch.index.codec.composite.composite912; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; @@ -55,14 +55,14 @@ * @opensearch.experimental */ @ExperimentalApi -public class Composite99DocValuesWriter extends DocValuesConsumer { +public class Composite912DocValuesWriter extends DocValuesConsumer { private final DocValuesConsumer delegate; private final SegmentWriteState state; private final MapperService mapperService; AtomicReference mergeState = new AtomicReference<>(); private final Set compositeMappedFieldTypes; private final Set compositeFieldSet; - private DocValuesConsumer composite99DocValuesConsumer; + private DocValuesConsumer compositeDocValuesConsumer; public IndexOutput dataOut; public IndexOutput metaOut; @@ -72,7 +72,7 @@ public class Composite99DocValuesWriter extends DocValuesConsumer { private final Map fieldProducerMap = new HashMap<>(); - public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) + public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) throws IOException { this.delegate = delegate; @@ -101,24 +101,24 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState // so that all the fields are sparse numeric doc values and not dense numeric doc values SegmentWriteState consumerWriteState = getSegmentWriteState(segmentWriteState); - this.composite99DocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + this.compositeDocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( consumerWriteState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); String dataFileName = IndexFileNames.segmentFileName( this.state.segmentInfo.name, this.state.segmentSuffix, - Composite99DocValuesFormat.DATA_EXTENSION + Composite912DocValuesFormat.DATA_EXTENSION ); dataOut = this.state.directory.createOutput(dataFileName, this.state.context); CodecUtil.writeIndexHeader( dataOut, - Composite99DocValuesFormat.DATA_CODEC_NAME, - Composite99DocValuesFormat.VERSION_CURRENT, + Composite912DocValuesFormat.DATA_CODEC_NAME, + Composite912DocValuesFormat.VERSION_CURRENT, this.state.segmentInfo.getId(), this.state.segmentSuffix ); @@ -126,13 +126,13 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState String metaFileName = IndexFileNames.segmentFileName( this.state.segmentInfo.name, this.state.segmentSuffix, - Composite99DocValuesFormat.META_EXTENSION + Composite912DocValuesFormat.META_EXTENSION ); metaOut = this.state.directory.createOutput(metaFileName, this.state.context); CodecUtil.writeIndexHeader( metaOut, - Composite99DocValuesFormat.META_CODEC_NAME, - Composite99DocValuesFormat.VERSION_CURRENT, + Composite912DocValuesFormat.META_CODEC_NAME, + Composite912DocValuesFormat.VERSION_CURRENT, this.state.segmentInfo.getId(), this.state.segmentSuffix ); @@ -197,12 +197,12 @@ public void close() throws IOException { success = true; } finally { if (success) { - IOUtils.close(dataOut, metaOut, composite99DocValuesConsumer); + IOUtils.close(dataOut, metaOut, compositeDocValuesConsumer); } else { - IOUtils.closeWhileHandlingException(dataOut, metaOut, composite99DocValuesConsumer); + IOUtils.closeWhileHandlingException(dataOut, metaOut, compositeDocValuesConsumer); } metaOut = dataOut = null; - composite99DocValuesConsumer = null; + compositeDocValuesConsumer = null; } } @@ -224,7 +224,7 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { - starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer); + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, compositeDocValuesConsumer); } } } @@ -313,7 +313,7 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { } } try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { - starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeSubsPerField, composite99DocValuesConsumer); + starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeSubsPerField, compositeDocValuesConsumer); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java deleted file mode 100644 index 20f1c7e9b7614..0000000000000 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.composite.composite99; - -import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.FilterCodec; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.mapper.MapperService; - -/** - * Extends the Codec to support new file formats for composite indices eg: star tree index - * based on the mappings. - * - * @opensearch.experimental - */ -@ExperimentalApi -public class Composite99Codec extends FilterCodec { - public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite99Codec"; - private final MapperService mapperService; - - // needed for SPI - this is used in reader path - public Composite99Codec() { - this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null); - } - - /** - * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to - * this ctor. - * - * @param name name of the codec - * @param delegate codec delegate - * @param mapperService mapper service instance - */ - protected Composite99Codec(String name, Codec delegate, MapperService mapperService) { - super(name, delegate); - this.mapperService = mapperService; - } - - @Override - public DocValuesFormat docValuesFormat() { - return new Composite99DocValuesFormat(mapperService); - } -} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/package-info.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/package-info.java deleted file mode 100644 index 3d6f130b9a7c8..0000000000000 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/package-info.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Responsible for handling all composite index codecs and operations associated with Composite99 codec - */ -package org.opensearch.index.codec.composite.composite99; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 255ad343cde32..003ebeafeae45 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -35,7 +35,7 @@ import java.util.Set; import java.util.function.Supplier; -import static org.opensearch.index.codec.composite.composite99.Composite99DocValuesReader.getSortedNumericDocValues; +import static org.opensearch.index.codec.composite.composite912.Composite912DocValuesReader.getSortedNumericDocValues; import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java index 8a5a4a5a94ce6..653907862b18c 100644 --- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java @@ -41,7 +41,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.ReplicationStats; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat; import org.opensearch.index.remote.RemoteSegmentStats; @@ -98,10 +98,10 @@ public class SegmentsStats implements Writeable, ToXContentFragment { Map.entry("tvd", "Term Vector Documents"), Map.entry("tvf", "Term Vector Fields"), Map.entry("liv", "Live Documents"), - Map.entry(Composite99DocValuesFormat.DATA_EXTENSION, "Composite Index"), - Map.entry(Composite99DocValuesFormat.META_EXTENSION, "Composite Index"), - Map.entry(Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, "Composite Index DocValues"), - Map.entry(Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION, "Composite Index DocValues"), + Map.entry(Composite912DocValuesFormat.DATA_EXTENSION, "Composite Index"), + Map.entry(Composite912DocValuesFormat.META_EXTENSION, "Composite Index"), + Map.entry(Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, "Composite Index DocValues"), + Map.entry(Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION, "Composite Index DocValues"), Map.entry(FuzzyFilterPostingsFormat.FUZZY_FILTER_FILE_EXTENSION, "Fuzzy Filter") ); diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 432d7e4906c08..16742fd33ed43 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,2 +1 @@ -org.opensearch.index.codec.composite.composite99.Composite99Codec org.opensearch.index.codec.composite.composite912.Composite912Codec diff --git a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java index 5abb01dd02e1b..b39dd77ea225e 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactoryTests.java @@ -17,7 +17,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -73,7 +73,7 @@ public void testGetDocValuesConsumerForCompositeCodec() throws IOException { ); assertEquals("org.apache.lucene.codecs.lucene90.Lucene90DocValuesConsumer", consumer.getClass().getName()); - assertEquals(CompositeCodecFactory.COMPOSITE_CODEC, Composite99Codec.COMPOSITE_INDEX_CODEC_NAME); + assertEquals(CompositeCodecFactory.COMPOSITE_CODEC, Composite912Codec.COMPOSITE_INDEX_CODEC_NAME); consumer.close(); } diff --git a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java index 67a953d923b22..7206828ff4723 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactoryTests.java @@ -19,7 +19,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -85,7 +85,7 @@ public void testGetDocValuesProducerForCompositeCodec99() throws IOException { newIOContext(random()) ); DocValuesProducer producer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( - Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + Composite912Codec.COMPOSITE_INDEX_CODEC_NAME, segmentReadState, dataCodec, dataExtension, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 705700aada2ff..012d04c575f55 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -25,7 +25,7 @@ import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -150,14 +150,14 @@ public static void setup() throws IOException { String dataFileName = IndexFileNames.segmentFileName( writeState.segmentInfo.name, writeState.segmentSuffix, - Composite99DocValuesFormat.DATA_EXTENSION + Composite912DocValuesFormat.DATA_EXTENSION ); dataOut = writeState.directory.createOutput(dataFileName, writeState.context); String metaFileName = IndexFileNames.segmentFileName( writeState.segmentInfo.name, writeState.segmentSuffix, - Composite99DocValuesFormat.META_EXTENSION + Composite912DocValuesFormat.META_EXTENSION ); metaOut = writeState.directory.createOutput(metaFileName, writeState.context); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java index bb31bd6a7cc27..cc6c1758697dd 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -26,8 +26,8 @@ import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; -import org.opensearch.index.codec.composite.composite99.Composite99Codec; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; @@ -351,12 +351,12 @@ public static void validateStarTreeFileFormats( ); DocValuesProducer compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec( - Composite99Codec.COMPOSITE_INDEX_CODEC_NAME, + Composite912Codec.COMPOSITE_INDEX_CODEC_NAME, readState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); IndexInput dataIn = readState.directory.openInput(dataFileName, IOContext.DEFAULT); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java index 095eda2986b3a..c4d6fe6f19812 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -28,7 +28,7 @@ import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; @@ -179,10 +179,10 @@ public void test_build_halfFloatMetrics() throws IOException { List metricsIterators = getMetricIterators(segmentStarTreeDocuments); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); @@ -298,10 +298,10 @@ public void test_build_floatMetrics() throws IOException { ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); @@ -391,10 +391,10 @@ public void test_build_longMetrics() throws IOException { List metricsIterators = getMetricIterators(segmentStarTreeDocuments); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( @@ -701,10 +701,10 @@ public void test_build() throws IOException { ); docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder.build(segmentStarTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); @@ -796,10 +796,10 @@ public void test_build_starTreeDataset() throws IOException { writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index 1aa830e3587df..a92ac39cb7020 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -16,7 +16,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -95,10 +95,10 @@ public void testFlushFlow() throws IOException { */ this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); List starTreeDocuments = builder.getStarTreeDocuments(); @@ -182,10 +182,10 @@ public void testFlushFlowDimsReverse() throws IOException { writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; @@ -284,10 +284,10 @@ public void testFlushFlowBuild() throws IOException { SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( consumerWriteState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); @@ -361,10 +361,10 @@ public void testFlushFlowWithTimestamps() throws IOException { ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(6, writeState.segmentInfo.getId()), mapperService); SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index f983365dfec30..00e53534a7606 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -14,7 +14,8 @@ import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; +import org.opensearch.index.compositeindex.CompositeIndexConstants; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -194,10 +195,10 @@ public void testMergeFlow() throws IOException { this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -310,10 +311,10 @@ public void testMergeFlowWithSum() throws IOException { writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -386,10 +387,10 @@ public void testMergeFlowWithCount() throws IOException { writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -528,10 +529,10 @@ public void testMergeFlowWithMissingDocs() throws IOException { writeState = getWriteState(4, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -615,10 +616,10 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { SegmentWriteState consumerWriteState = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( consumerWriteState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -702,10 +703,10 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException writeState = getWriteState(4, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -793,10 +794,10 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { writeState = getWriteState(4, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -879,10 +880,10 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -957,10 +958,10 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { writeState = getWriteState(0, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -1086,10 +1087,10 @@ public void testMergeFlowWithDuplicateDimensionValues() throws IOException { ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); @@ -1234,10 +1235,10 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); @@ -1352,10 +1353,10 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { writeState = getWriteState(4, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -1489,10 +1490,10 @@ public void testMergeFlowWithDuplicateDimensionValueWithMaxLeafDocs() throws IOE ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); @@ -1618,10 +1619,10 @@ public void testMergeFlowWithMaxLeafDocsAndStarTreeNodesAssertion() throws IOExc ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); builder.build(builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)), new AtomicInteger(), docValuesConsumer); @@ -1727,10 +1728,10 @@ public void testMergeFlowWithTimestamps() throws IOException { ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, - Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, - Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, - Composite99DocValuesFormat.META_DOC_VALUES_CODEC, - Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); @@ -1850,7 +1851,7 @@ private StarTreeValues getStarTreeValues( null, dimDocIdSetIterators, metricDocIdSetIterators, - Map.of(SEGMENT_DOCS_COUNT, number), + Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number), null ); return starTreeValues; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java index 4c854f7546197..6733cac12f657 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -26,7 +26,7 @@ import org.apache.lucene.util.NumericUtils; import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat; +import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.CompositeIndexConstants; import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.DateDimension; @@ -158,14 +158,14 @@ public void setup() throws IOException { dataFileName = IndexFileNames.segmentFileName( writeState.segmentInfo.name, writeState.segmentSuffix, - Composite99DocValuesFormat.DATA_EXTENSION + Composite912DocValuesFormat.DATA_EXTENSION ); dataOut = writeState.directory.createOutput(dataFileName, writeState.context); metaFileName = IndexFileNames.segmentFileName( writeState.segmentInfo.name, writeState.segmentSuffix, - Composite99DocValuesFormat.META_EXTENSION + Composite912DocValuesFormat.META_EXTENSION ); metaOut = writeState.directory.createOutput(metaFileName, writeState.context); From acf209fc2a23d4f7220b1480c9e407e6cf3d8c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 10:30:05 +0800 Subject: [PATCH 206/357] Bump com.google.api-client:google-api-client from 2.2.0 to 2.7.0 in /plugins/repository-gcs (#16216) * Bump com.google.api-client:google-api-client in /plugins/repository-gcs Bumps [com.google.api-client:google-api-client](https://github.com/googleapis/google-api-java-client) from 2.2.0 to 2.7.0. - [Release notes](https://github.com/googleapis/google-api-java-client/releases) - [Changelog](https://github.com/googleapis/google-api-java-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-api-java-client/compare/v2.2.0...v2.7.0) --- updated-dependencies: - dependency-name: com.google.api-client:google-api-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 | 1 - .../repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cad8c764a8e5..510fb83b42978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) +- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) ### Changed diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 94d25af94e67f..904b5d8f08426 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -58,7 +58,7 @@ dependencies { api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0' - api 'com.google.api-client:google-api-client:2.2.0' + api 'com.google.api-client:google-api-client:2.7.0' api 'com.google.api.grpc:proto-google-common-protos:2.37.1' api 'com.google.api.grpc:proto-google-iam-v1:1.33.0' diff --git a/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 deleted file mode 100644 index f9604d6837ca9..0000000000000 --- a/plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10e53fd4d987e37190432e896bdaa62e8ea2c628 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 new file mode 100644 index 0000000000000..dcbd27a0009bf --- /dev/null +++ b/plugins/repository-gcs/licenses/google-api-client-2.7.0.jar.sha1 @@ -0,0 +1 @@ +59c8e5e3c03f146561a83051af3ca945d40e02c6 \ No newline at end of file From e7757e76837805148b9ca271d05f50cf83e2b025 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 9 Oct 2024 11:18:52 -0400 Subject: [PATCH 207/357] Update Apache Lucene version for 2.18.0 (#16252) Signed-off-by: Andriy Redko --- libs/core/src/main/java/org/opensearch/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 5d38f85c40c89..6b1f6898eec9b 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -111,7 +111,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1); - public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1); + public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 9225aa270d15d4c49327c70b3780b96895e364c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:24:32 -0400 Subject: [PATCH 208/357] Bump com.squareup.okio:okio from 3.9.0 to 3.9.1 in /test/fixtures/hdfs-fixture (#16212) * Bump com.squareup.okio:okio in /test/fixtures/hdfs-fixture Bumps [com.squareup.okio:okio](https://github.com/square/okio) from 3.9.0 to 3.9.1. - [Release notes](https://github.com/square/okio/releases) - [Changelog](https://github.com/square/okio/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okio/compare/parent-3.9.0...3.9.1) --- updated-dependencies: - dependency-name: com.squareup.okio:okio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 510fb83b42978..100936291c932 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) - Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) +- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index bc63362980bea..b3a90b5de2589 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -88,6 +88,6 @@ dependencies { runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") { exclude group: "com.squareup.okio" } - runtimeOnly "com.squareup.okio:okio:3.9.0" + runtimeOnly "com.squareup.okio:okio:3.9.1" runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.6" } From ba0ccfa50828688ea5793fc1f1fd08c6a21d5f92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:41:30 -0400 Subject: [PATCH 209/357] Bump io.grpc:grpc-api from 1.57.2 to 1.68.0 in /plugins/discovery-gce (#16213) * Bump io.grpc:grpc-api from 1.57.2 to 1.68.0 in /plugins/discovery-gce Bumps [io.grpc:grpc-api](https://github.com/grpc/grpc-java) from 1.57.2 to 1.68.0. - [Release notes](https://github.com/grpc/grpc-java/releases) - [Commits](https://github.com/grpc/grpc-java/compare/v1.57.2...v1.68.0) --- updated-dependencies: - dependency-name: io.grpc:grpc-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] * Centralize grpc version management Signed-off-by: Andriy Redko --------- Signed-off-by: dependabot[bot] Signed-off-by: Andriy Redko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 1 + plugins/discovery-gce/build.gradle | 2 +- plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 | 1 - plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1 | 1 + plugins/repository-gcs/build.gradle | 2 +- plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1 | 1 - plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1 | 1 + 8 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 100936291c932..cae61bf239a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) - Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) +- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) - Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) ### Changed diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 19e2ba2e2c5bb..9affa9b9a45f1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -25,6 +25,7 @@ jakarta_annotation = 1.3.5 google_http_client = 1.44.1 tdigest = 3.3 hdrhistogram = 2.2.2 +grpc = 1.68.0 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 80aae03bc0332..a08fa1d968e30 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -30,7 +30,7 @@ dependencies { api "commons-logging:commons-logging:${versions.commonslogging}" api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api "commons-codec:commons-codec:${versions.commonscodec}" - api 'io.grpc:grpc-api:1.57.2' + api "io.grpc:grpc-api:${versions.grpc}" api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' runtimeOnly "com.google.guava:guava:${versions.guava}" diff --git a/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 b/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 deleted file mode 100644 index 8b320fdd2f9cc..0000000000000 --- a/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c71a006b81ddae7bc4b7cb1d2da78c1b173761f4 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1 b/plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1 new file mode 100644 index 0000000000000..bf45716c5b8ce --- /dev/null +++ b/plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1 @@ -0,0 +1 @@ +9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08 \ No newline at end of file diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 904b5d8f08426..ab129ab7f116a 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -86,7 +86,7 @@ dependencies { api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api "commons-codec:commons-codec:${versions.commonscodec}" api 'org.threeten:threetenbp:1.4.4' - api 'io.grpc:grpc-api:1.57.2' + api "io.grpc:grpc-api:${versions.grpc}" api 'io.opencensus:opencensus-api:0.31.1' api 'io.opencensus:opencensus-contrib-http-util:0.31.1' diff --git a/plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1 b/plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1 deleted file mode 100644 index 8b320fdd2f9cc..0000000000000 --- a/plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c71a006b81ddae7bc4b7cb1d2da78c1b173761f4 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1 b/plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1 new file mode 100644 index 0000000000000..bf45716c5b8ce --- /dev/null +++ b/plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1 @@ -0,0 +1 @@ +9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08 \ No newline at end of file From a09750a5858420c08a7af4c2a42c7c95f1fb7f41 Mon Sep 17 00:00:00 2001 From: Finn Date: Wed, 9 Oct 2024 11:43:59 -0700 Subject: [PATCH 210/357] Remove redundant force merge - histo rest tests (#16144) Signed-off-by: Finn Carroll --- .../rest-api-spec/test/search.aggregation/10_histogram.yml | 5 ----- .../rest-api-spec/test/search.aggregation/230_composite.yml | 5 ----- .../test/search.aggregation/330_auto_date_histogram.yml | 5 ----- .../test/search.aggregation/360_date_histogram.yml | 5 ----- .../rest-api-spec/test/search.aggregation/40_range.yml | 5 ----- 5 files changed, 25 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index a75b1d0eac793..996c2aae8cfe4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -678,11 +678,6 @@ setup: - '{"index": {}}' - '{"date": "2016-03-01"}' - - do: - indices.forcemerge: - index: test_2 - max_num_segments: 1 - - do: search: index: test_2 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index ade9eb3eee0dc..408e3c9fe0f9f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -1102,11 +1102,6 @@ setup: - '{"index": {}}' - '{"date": "2016-03-01"}' - - do: - indices.forcemerge: - index: test_2 - max_num_segments: 1 - - do: search: index: test_2 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml index 0897e0bdd894b..fc82517788c91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml @@ -133,11 +133,6 @@ setup: - '{"index": {}}' - '{"date": "2020-03-09", "v": 4}' - - do: - indices.forcemerge: - index: test_profile - max_num_segments: 1 - - do: search: index: test_profile diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml index 0cabbf6f25133..fe143653c0674 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml @@ -140,11 +140,6 @@ setup: - '{"index": {}}' - '{"date": "2025-02-14"}' - - do: - indices.forcemerge: - index: dhisto-agg-w-query - max_num_segments: 1 - - do: search: index: dhisto-agg-w-query diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index 93ca668f93a6f..2cc49061d53e8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -568,11 +568,6 @@ setup: - '{"index": {}}' - '{"double" : 50}' - - do: - indices.forcemerge: - index: test_profile - max_num_segments: 1 - - do: search: index: test_profile From 68e3e45a8e5874ce1adef78a506c25b57465208c Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 9 Oct 2024 15:56:44 -0400 Subject: [PATCH 211/357] The protobuf-java leaks through client library dependencies (#16254) * The protobuf-java leaks through client library dependencies Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + client/rest-high-level/build.gradle | 4 +++- .../rest/action/document/RestBulkStreamingAction.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cae61bf239a97..0067d69837eed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) +- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) ### Security diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index fdc93d8037ce6..b08df5bf09bac 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -50,7 +50,9 @@ restResources { } dependencies { - api project(':server') + api(project(':server')) { + exclude group: 'com.google.protobuf' + } api project(':client:rest') api project(':modules:mapper-extras') api project(':modules:parent-join') diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestBulkStreamingAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestBulkStreamingAction.java index 2e0d1b8ead814..42b4ed3ef3b84 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestBulkStreamingAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestBulkStreamingAction.java @@ -8,7 +8,6 @@ package org.opensearch.rest.action.document; -import com.google.protobuf.ExperimentalApi; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.bulk.BulkItemResponse; @@ -18,6 +17,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.client.Requests; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -59,7 +59,7 @@ * { "type1" : { "field1" : "value1" } } * * - * @opensearch.api + * @opensearch.experimental */ @ExperimentalApi public class RestBulkStreamingAction extends BaseRestHandler { From 9e53d9b95124273d862ce4a06db3d6751addfa46 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Thu, 10 Oct 2024 11:47:44 +0800 Subject: [PATCH 212/357] Fix Flaky Test org.opensearch.gateway.RecoveryFromGatewayIT.testShardStoreFetchMultiNodeMultiIndexesUsingBatchAction (#16093) Signed-off-by: kkewwei --- .../gateway/RecoveryFromGatewayIT.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java index bcf23a37c0010..8d0a3b5788a70 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java @@ -1366,7 +1366,8 @@ public void testSingleShardStoreFetchUsingBatchAction() throws ExecutionExceptio DiscoveryNode[] nodes = getDiscoveryNodes(); TransportNodesListShardStoreMetadataBatch.NodesStoreFilesMetadataBatch response = prepareAndSendRequest( new String[] { indexName }, - nodes + nodes, + false ); Index index = resolveIndex(indexName); ShardId shardId = new ShardId(index, 0); @@ -1379,12 +1380,14 @@ public void testSingleShardStoreFetchUsingBatchAction() throws ExecutionExceptio public void testShardStoreFetchMultiNodeMultiIndexesUsingBatchAction() throws Exception { internalCluster().startNodes(2); + ensureStableCluster(2); String indexName1 = "test1"; String indexName2 = "test2"; DiscoveryNode[] nodes = getDiscoveryNodes(); TransportNodesListShardStoreMetadataBatch.NodesStoreFilesMetadataBatch response = prepareAndSendRequest( new String[] { indexName1, indexName2 }, - nodes + nodes, + true ); ClusterSearchShardsResponse searchShardsResponse = client().admin().cluster().prepareSearchShards(indexName1, indexName2).get(); for (ClusterSearchShardsGroup clusterSearchShardsGroup : searchShardsResponse.getGroups()) { @@ -1406,7 +1409,8 @@ public void testShardStoreFetchNodeNotConnectedUsingBatchAction() { String indexName = "test"; TransportNodesListShardStoreMetadataBatch.NodesStoreFilesMetadataBatch response = prepareAndSendRequest( new String[] { indexName }, - new DiscoveryNode[] { nonExistingNode } + new DiscoveryNode[] { nonExistingNode }, + false ); assertTrue(response.hasFailures()); assertEquals(1, response.failures().size()); @@ -1513,10 +1517,14 @@ private void prepareIndices(String[] indices, int numberOfPrimaryShards, int num private TransportNodesListShardStoreMetadataBatch.NodesStoreFilesMetadataBatch prepareAndSendRequest( String[] indices, - DiscoveryNode[] nodes + DiscoveryNode[] nodes, + boolean ensureGreen ) { Map shardAttributesMap = null; prepareIndices(indices, 1, 1); + if (ensureGreen) { + ensureGreen(indices); + } shardAttributesMap = prepareRequestMap(indices, 1); TransportNodesListShardStoreMetadataBatch.NodesStoreFilesMetadataBatch response; return ActionTestUtils.executeBlocking( From d7b011612014a78283c56425d493550b64ad2b5b Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:09:17 +0530 Subject: [PATCH 213/357] Revert remote publication method renaming in DiscoveryNode (#16250) Signed-off-by: Sooraj Sinha --- .../opensearch/cluster/coordination/CoordinationState.java | 2 +- .../org/opensearch/cluster/coordination/JoinTaskExecutor.java | 4 ++-- .../cluster/coordination/PublicationTransportHandler.java | 2 +- .../main/java/org/opensearch/cluster/node/DiscoveryNode.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index e5b07f867d609..9cffc7051d756 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -108,7 +108,7 @@ public CoordinationState( // ToDo: revisit this check while making the setting dynamic this.isRemotePublicationEnabled = isRemoteStateEnabled && REMOTE_PUBLICATION_SETTING.get(settings) - && localNode.isRemoteStatePublicationConfigured(); + && localNode.isRemoteStatePublicationEnabled(); } public boolean isRemotePublicationEnabled() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index aca8653be4dd8..13033b670d44b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -513,10 +513,10 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi assert existingNodes.isEmpty() == false; Optional remotePublicationNode = existingNodes.stream() - .filter(DiscoveryNode::isRemoteStatePublicationConfigured) + .filter(DiscoveryNode::isRemoteStatePublicationEnabled) .findFirst(); - if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationConfigured()) { + if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 6277b70f9a7c8..42aa55433dd5f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -370,7 +370,7 @@ private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes dis assert ClusterMetadataManifest.getCodecForVersion(discoveryNodes.getMinNodeVersion()) >= ClusterMetadataManifest.CODEC_V0; for (DiscoveryNode node : discoveryNodes.getNodes().values()) { // if a node is non-remote then created local publication context - if (node.isRemoteStatePublicationConfigured() == false) { + if (node.isRemoteStatePublicationEnabled() == false) { return false; } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index b06c2ef0bdbe4..8c9a37a767ede 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -518,7 +518,7 @@ public boolean isRemoteStoreNode() { * Returns whether settings required for remote cluster state publication is configured * @return true if the node contains remote cluster state node attribute and remote routing table node attribute */ - public boolean isRemoteStatePublicationConfigured() { + public boolean isRemoteStatePublicationEnabled() { return this.getAttributes() .keySet() .stream() From 58adc182c7e0e0a32c77bb3b43ce54939355320a Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 10 Oct 2024 10:10:23 +0530 Subject: [PATCH 214/357] BugFix: call listener.onFailure on failure to pin the timestamp (#16248) Signed-off-by: Sachin Kale --- .../RemoteStorePinnedTimestampsIT.java | 128 ++++++++++++++++-- .../RemoteStorePinnedTimestampService.java | 24 ++-- 2 files changed, 131 insertions(+), 21 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java index 3e1127e0ce240..ee51eff4e1bd5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java @@ -11,21 +11,29 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; +import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; +import org.opensearch.repositories.fs.ReloadableFsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.REMOTE_STORE; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_SLOWDOWN_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase { @@ -33,18 +41,29 @@ public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase @Override protected Settings nodeSettings(int nodeOrdinal) { + String segmentRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) + .put(segmentRepoTypeAttributeKey, ReloadableFsRepository.TYPE) .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true) .build(); } ActionListener noOpActionListener = new ActionListener<>() { @Override - public void onResponse(Void unused) {} + public void onResponse(Void unused) { + // do nothing + } @Override - public void onFailure(Exception e) {} + public void onFailure(Exception e) { + fail(); + } }; public void testTimestampPinUnpin() throws Exception { @@ -57,15 +76,8 @@ public void testTimestampPinUnpin() throws Exception { ); Tuple> pinnedTimestampWithFetchTimestamp = RemoteStorePinnedTimestampService.getPinnedTimestamps(); - long lastFetchTimestamp = pinnedTimestampWithFetchTimestamp.v1(); - assertEquals(-1L, lastFetchTimestamp); assertEquals(Set.of(), pinnedTimestampWithFetchTimestamp.v2()); - assertThrows( - IllegalArgumentException.class, - () -> remoteStorePinnedTimestampService.pinTimestamp(1234L, "ss1", noOpActionListener) - ); - long timestamp1 = System.currentTimeMillis() + 30000L; long timestamp2 = System.currentTimeMillis() + 60000L; long timestamp3 = System.currentTimeMillis() + 900000L; @@ -197,6 +209,104 @@ public void onFailure(Exception e) { remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueMinutes(3)); } + public void testPinExceptionsOlderTimestamp() throws InterruptedException { + prepareCluster(1, 1, INDEX_NAME, 0, 2); + ensureGreen(INDEX_NAME); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + CountDownLatch latch = new CountDownLatch(1); + remoteStorePinnedTimestampService.pinTimestamp(1234L, "ss1", new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + // We expect onFailure to be called + fail(); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + }, latch)); + + latch.await(); + } + + public void testPinExceptionsRemoteStoreCallTakeTime() throws InterruptedException, ExecutionException { + prepareCluster(1, 1, INDEX_NAME, 0, 2); + ensureGreen(INDEX_NAME); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + CountDownLatch latch = new CountDownLatch(1); + slowDownRepo(REPOSITORY_NAME, 10); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueSeconds(1)); + long timestampToBePinned = System.currentTimeMillis() + 600000; + remoteStorePinnedTimestampService.pinTimestamp(timestampToBePinned, "ss1", new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + // We expect onFailure to be called + fail(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e.getMessage()); + assertTrue(e instanceof RuntimeException); + assertTrue(e.getMessage().contains("Timestamp pinning took")); + + // Check if the timestamp was unpinned + remoteStorePinnedTimestampService.forceSyncPinnedTimestamps(); + assertFalse(RemoteStorePinnedTimestampService.getPinnedTimestamps().v2().contains(timestampToBePinned)); + } + }, latch)); + + latch.await(); + } + + protected void slowDownRepo(String repoName, int value) throws ExecutionException, InterruptedException { + GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName }); + GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get(); + RepositoryMetadata rmd = res.repositories().get(0); + Settings.Builder settings = Settings.builder() + .put("location", rmd.settings().get("location")) + .put(REPOSITORIES_SLOWDOWN_SETTING.getKey(), value); + createRepository(repoName, rmd.type(), settings); + } + + public void testUnpinException() throws InterruptedException { + prepareCluster(1, 1, INDEX_NAME, 0, 2); + ensureGreen(INDEX_NAME); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(INDEX_NAME) + ); + + CountDownLatch latch = new CountDownLatch(1); + remoteStorePinnedTimestampService.unpinTimestamp(1234L, "dummy-entity", new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(Void unused) { + // We expect onFailure to be called + fail(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e.getMessage()); + assertTrue(e instanceof IllegalArgumentException); + } + }, latch)); + + latch.await(); + } + public void testLastSuccessfulFetchOfPinnedTimestampsPresentInNodeStats() throws Exception { logger.info("Starting up cluster manager"); logger.info("cluster.remote_store.pinned_timestamps.enabled set to true"); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index a3382d8568ec5..98fcad0e6c496 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -130,16 +130,16 @@ public static Map> fetchPinnedTimestamps(Settings settings, Re * @throws IllegalArgumentException If the timestamp is less than the current time minus one second */ public void pinTimestamp(long timestamp, String pinningEntity, ActionListener listener) { - // If a caller uses current system time to pin the timestamp, following check will almost always fail. - // So, we allow pinning timestamp in the past upto some buffer - long lookbackIntervalInMills = RemoteStoreSettings.getPinnedTimestampsLookbackInterval().millis(); - if (timestamp < (System.currentTimeMillis() - lookbackIntervalInMills)) { - throw new IllegalArgumentException( - "Timestamp to be pinned is less than current timestamp - value of cluster.remote_store.pinned_timestamps.lookback_interval" - ); - } - long startTime = System.nanoTime(); try { + // If a caller uses current system time to pin the timestamp, following check will almost always fail. + // So, we allow pinning timestamp in the past upto some buffer + long lookbackIntervalInMills = RemoteStoreSettings.getPinnedTimestampsLookbackInterval().millis(); + if (timestamp < (System.currentTimeMillis() - lookbackIntervalInMills)) { + throw new IllegalArgumentException( + "Timestamp to be pinned is less than current timestamp - value of cluster.remote_store.pinned_timestamps.lookback_interval" + ); + } + long startTime = System.nanoTime(); logger.debug("Pinning timestamp = {} against entity = {}", timestamp, pinningEntity); blobContainer.writeBlob(getBlobName(timestamp, pinningEntity), new ByteArrayInputStream(new byte[0]), 0, true); long elapsedTime = System.nanoTime() - startTime; @@ -155,7 +155,7 @@ public void pinTimestamp(long timestamp, String pinningEntity, ActionListener Date: Thu, 10 Oct 2024 00:28:11 -0700 Subject: [PATCH 215/357] [Workload Management] QueryGroup Stats API Logic (#15777) * changelog Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * add tests Signed-off-by: Ruirui Zhang * modify uri Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * changelog Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * add tests Signed-off-by: Ruirui Zhang * modify uri Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * revise Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * changelog Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * add tests Signed-off-by: Ruirui Zhang * modify uri Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * git pull Signed-off-by: Ruirui Zhang * rebase Signed-off-by: Ruirui Zhang * encapsulate querygroupstats in wlmstats Signed-off-by: Ruirui Zhang * fix UT Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + .../org/opensearch/action/ActionModule.java | 5 + .../cluster/wlm/TransportWlmStatsAction.java | 79 ++++++++++++++ .../admin/cluster/wlm/WlmStatsAction.java | 25 +++++ .../admin/cluster/wlm/WlmStatsRequest.java | 65 +++++++++++ .../admin/cluster/wlm/WlmStatsResponse.java | 73 +++++++++++++ .../admin/cluster/wlm/package-info.java | 10 ++ .../opensearch/client/ClusterAdminClient.java | 9 ++ .../client/support/AbstractClient.java | 8 ++ .../main/java/org/opensearch/node/Node.java | 12 +- .../admin/cluster/RestWlmStatsAction.java | 58 ++++++++++ .../org/opensearch/wlm/QueryGroupService.java | 47 ++++++-- .../opensearch/wlm/stats/QueryGroupStats.java | 4 + .../org/opensearch/wlm/stats/WlmStats.java | 65 +++++++++++ .../cluster/wlm/WlmStatsResponseTests.java | 103 ++++++++++++++++++ .../nodes/TransportWlmStatsActionTests.java | 80 ++++++++++++++ ...eryGroupRequestOperationListenerTests.java | 14 ++- .../wlm/stats/QueryGroupStatsTests.java | 14 +++ .../opensearch/wlm/stats/WlmStatsTests.java | 75 +++++++++++++ 19 files changed, 731 insertions(+), 16 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/wlm/TransportWlmStatsAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsRequest.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponse.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/wlm/package-info.java create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/cluster/RestWlmStatsAction.java create mode 100644 server/src/main/java/org/opensearch/wlm/stats/WlmStats.java create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java create mode 100644 server/src/test/java/org/opensearch/action/support/nodes/TransportWlmStatsActionTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0067d69837eed..5d295c9a199f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) +- [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777)) - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110)) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 12c7626e385a4..1e6eae87af53a 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -123,6 +123,8 @@ import org.opensearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksAction; import org.opensearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction; +import org.opensearch.action.admin.cluster.wlm.TransportWlmStatsAction; +import org.opensearch.action.admin.cluster.wlm.WlmStatsAction; import org.opensearch.action.admin.indices.alias.IndicesAliasesAction; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.admin.indices.alias.TransportIndicesAliasesAction; @@ -375,6 +377,7 @@ import org.opensearch.rest.action.admin.cluster.RestRestoreSnapshotAction; import org.opensearch.rest.action.admin.cluster.RestSnapshotsStatusAction; import org.opensearch.rest.action.admin.cluster.RestVerifyRepositoryAction; +import org.opensearch.rest.action.admin.cluster.RestWlmStatsAction; import org.opensearch.rest.action.admin.cluster.dangling.RestDeleteDanglingIndexAction; import org.opensearch.rest.action.admin.cluster.dangling.RestImportDanglingIndexAction; import org.opensearch.rest.action.admin.cluster.dangling.RestListDanglingIndicesAction; @@ -622,6 +625,7 @@ public void reg actions.register(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class); actions.register(RemoteInfoAction.INSTANCE, TransportRemoteInfoAction.class); actions.register(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class); + actions.register(WlmStatsAction.INSTANCE, TransportWlmStatsAction.class); actions.register(RemoteStoreStatsAction.INSTANCE, TransportRemoteStoreStatsAction.class); actions.register(NodesUsageAction.INSTANCE, TransportNodesUsageAction.class); actions.register(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class); @@ -828,6 +832,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestClearVotingConfigExclusionsAction()); registerHandler.accept(new RestMainAction()); registerHandler.accept(new RestNodesInfoAction(settingsFilter)); + registerHandler.accept(new RestWlmStatsAction()); registerHandler.accept(new RestRemoteClusterInfoAction()); registerHandler.accept(new RestNodesStatsAction()); registerHandler.accept(new RestNodesUsageAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/wlm/TransportWlmStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/TransportWlmStatsAction.java new file mode 100644 index 0000000000000..9c2fb3f1689ec --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/TransportWlmStatsAction.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.wlm; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.nodes.TransportNodesAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.opensearch.wlm.QueryGroupService; +import org.opensearch.wlm.stats.WlmStats; + +import java.io.IOException; +import java.util.List; + +/** + * Transport action for obtaining WlmStats + * + * @opensearch.experimental + */ +public class TransportWlmStatsAction extends TransportNodesAction { + + final QueryGroupService queryGroupService; + + @Inject + public TransportWlmStatsAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + QueryGroupService queryGroupService, + ActionFilters actionFilters + ) { + super( + WlmStatsAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + WlmStatsRequest::new, + WlmStatsRequest::new, + ThreadPool.Names.MANAGEMENT, + WlmStats.class + ); + this.queryGroupService = queryGroupService; + } + + @Override + protected WlmStatsResponse newResponse(WlmStatsRequest request, List wlmStats, List failures) { + return new WlmStatsResponse(clusterService.getClusterName(), wlmStats, failures); + } + + @Override + protected WlmStatsRequest newNodeRequest(WlmStatsRequest request) { + return request; + } + + @Override + protected WlmStats newNodeResponse(StreamInput in) throws IOException { + return new WlmStats(in); + } + + @Override + protected WlmStats nodeOperation(WlmStatsRequest wlmStatsRequest) { + assert transportService.getLocalNode() != null; + return new WlmStats( + transportService.getLocalNode(), + queryGroupService.nodeStats(wlmStatsRequest.getQueryGroupIds(), wlmStatsRequest.isBreach()) + ); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsAction.java new file mode 100644 index 0000000000000..2dfb10fc5dc90 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsAction.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.wlm; + +import org.opensearch.action.ActionType; + +/** + * Transport action for obtaining Workload Management Stats. + * + * @opensearch.experimental + */ +public class WlmStatsAction extends ActionType { + public static final WlmStatsAction INSTANCE = new WlmStatsAction(); + public static final String NAME = "cluster:monitor/wlm/stats"; + + private WlmStatsAction() { + super(NAME, WlmStatsResponse::new); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsRequest.java new file mode 100644 index 0000000000000..bf4f79faff478 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsRequest.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.wlm; + +import org.opensearch.action.support.nodes.BaseNodesRequest; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * A request to get Workload Management Stats + */ +@ExperimentalApi +public class WlmStatsRequest extends BaseNodesRequest { + + private final Set queryGroupIds; + private final Boolean breach; + + public WlmStatsRequest(StreamInput in) throws IOException { + super(in); + this.queryGroupIds = new HashSet<>(Set.of(in.readStringArray())); + this.breach = in.readOptionalBoolean(); + } + + /** + * Get QueryGroup stats from nodes based on the nodes ids specified. If none are passed, stats + * for all nodes will be returned. + */ + public WlmStatsRequest(String[] nodesIds, Set queryGroupIds, Boolean breach) { + super(false, nodesIds); + this.queryGroupIds = queryGroupIds; + this.breach = breach; + } + + public WlmStatsRequest() { + super(false, (String[]) null); + queryGroupIds = new HashSet<>(); + this.breach = false; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(queryGroupIds.toArray(new String[0])); + out.writeOptionalBoolean(breach); + } + + public Set getQueryGroupIds() { + return queryGroupIds; + } + + public Boolean isBreach() { + return breach; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponse.java new file mode 100644 index 0000000000000..2ce1b09a61fc6 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponse.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.wlm; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterName; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.wlm.stats.QueryGroupStats; +import org.opensearch.wlm.stats.WlmStats; + +import java.io.IOException; +import java.util.List; + +/** + * A response for obtaining Workload Management Stats + */ +@ExperimentalApi +public class WlmStatsResponse extends BaseNodesResponse implements ToXContentFragment { + + WlmStatsResponse(StreamInput in) throws IOException { + super(in); + } + + WlmStatsResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(WlmStats::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (WlmStats wlmStats : getNodes()) { + builder.startObject(wlmStats.getNode().getId()); + QueryGroupStats queryGroupStats = wlmStats.getQueryGroupStats(); + queryGroupStats.toXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.toString(); + } catch (IOException e) { + return "{ \"error\" : \"" + e.getMessage() + "\"}"; + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/wlm/package-info.java b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/package-info.java new file mode 100644 index 0000000000000..13724b335e7c6 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/wlm/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** WlmStats transport handlers. */ +package org.opensearch.action.admin.cluster.wlm; diff --git a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java index 05f09c1a6e661..5ce4d442fbe0b 100644 --- a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java @@ -137,6 +137,8 @@ import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.opensearch.action.admin.cluster.wlm.WlmStatsRequest; +import org.opensearch.action.admin.cluster.wlm.WlmStatsResponse; import org.opensearch.action.admin.indices.dangling.delete.DeleteDanglingIndexRequest; import org.opensearch.action.admin.indices.dangling.import_index.ImportDanglingIndexRequest; import org.opensearch.action.admin.indices.dangling.list.ListDanglingIndicesRequest; @@ -320,6 +322,13 @@ public interface ClusterAdminClient extends OpenSearchClient { */ NodesStatsRequestBuilder prepareNodesStats(String... nodesIds); + /** + * QueryGroup stats of the cluster. + * @param request The wlmStatsRequest + * @param listener A listener to be notified with a result + */ + void wlmStats(WlmStatsRequest request, ActionListener listener); + void remoteStoreStats(RemoteStoreStatsRequest request, ActionListener listener); RemoteStoreStatsRequestBuilder prepareRemoteStoreStats(String index, String shardId); diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index 509cd732357d6..f4683ab516cef 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -179,6 +179,9 @@ import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksResponse; +import org.opensearch.action.admin.cluster.wlm.WlmStatsAction; +import org.opensearch.action.admin.cluster.wlm.WlmStatsRequest; +import org.opensearch.action.admin.cluster.wlm.WlmStatsResponse; import org.opensearch.action.admin.indices.alias.IndicesAliasesAction; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; @@ -918,6 +921,11 @@ public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { return new NodesStatsRequestBuilder(this, NodesStatsAction.INSTANCE).setNodesIds(nodesIds); } + @Override + public void wlmStats(final WlmStatsRequest request, final ActionListener listener) { + execute(WlmStatsAction.INSTANCE, request, listener); + } + @Override public void remoteStoreStats(final RemoteStoreStatsRequest request, final ActionListener listener) { execute(RemoteStoreStatsAction.INSTANCE, request, listener); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 5c965b06a4b69..584d95b9ff6b5 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -354,12 +354,12 @@ public class Node implements Closeable { ); /** - * controls whether the node is allowed to persist things like metadata to disk - * Note that this does not control whether the node stores actual indices (see - * {@link #NODE_DATA_SETTING}). However, if this is false, {@link #NODE_DATA_SETTING} - * and {@link #NODE_MASTER_SETTING} must also be false. - * - */ + * controls whether the node is allowed to persist things like metadata to disk + * Note that this does not control whether the node stores actual indices (see + * {@link #NODE_DATA_SETTING}). However, if this is false, {@link #NODE_DATA_SETTING} + * and {@link #NODE_MASTER_SETTING} must also be false. + * + */ public static final Setting NODE_LOCAL_STORAGE_SETTING = Setting.boolSetting( "node.local_storage", true, diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestWlmStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestWlmStatsAction.java new file mode 100644 index 0000000000000..51bd313e74df0 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestWlmStatsAction.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.cluster; + +import org.opensearch.action.admin.cluster.wlm.WlmStatsRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.common.Strings; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestActions; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * Transport action to get Workload Management stats + * + * @opensearch.experimental + */ +public class RestWlmStatsAction extends BaseRestHandler { + + @Override + public List routes() { + return unmodifiableList( + asList( + new Route(GET, "_wlm/stats"), + new Route(GET, "_wlm/{nodeId}/stats"), + new Route(GET, "_wlm/stats/{queryGroupId}"), + new Route(GET, "_wlm/{nodeId}/stats/{queryGroupId}") + ) + ); + } + + @Override + public String getName() { + return "wlm_stats_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); + Set queryGroupIds = Strings.tokenizeByCommaToSet(request.param("queryGroupId", "_all")); + Boolean breach = request.hasParam("breach") ? Boolean.parseBoolean(request.param("boolean")) : null; + WlmStatsRequest wlmStatsRequest = new WlmStatsRequest(nodesIds, queryGroupIds, breach); + return channel -> client.admin().cluster().wlmStats(wlmStatsRequest, new RestActions.NodesResponseRestListener<>(channel)); + } +} diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java index cda5916db26f3..30d39686931be 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.search.SearchShardTask; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; @@ -42,6 +43,7 @@ /** * As of now this is a stub and main implementation PR will be raised soon.Coming PR will collate these changes with core QueryGroupService changes + * @opensearch.experimental */ public class QueryGroupService extends AbstractLifecycleComponent implements @@ -49,7 +51,6 @@ public class QueryGroupService extends AbstractLifecycleComponent TaskResourceTrackingService.TaskCompletionListener { private static final Logger logger = LogManager.getLogger(QueryGroupService.class); - private final QueryGroupTaskCancellationService taskCancellationService; private volatile Scheduler.Cancellable scheduledFuture; private final ThreadPool threadPool; @@ -205,16 +206,48 @@ public void incrementFailuresFor(final String queryGroupId) { /** * @return node level query group stats */ - public QueryGroupStats nodeStats() { + public QueryGroupStats nodeStats(Set queryGroupIds, Boolean requestedBreached) { final Map statsHolderMap = new HashMap<>(); - for (Map.Entry queryGroupsState : queryGroupsStateAccessor.getQueryGroupStateMap().entrySet()) { - final String queryGroupId = queryGroupsState.getKey(); - final QueryGroupState currentState = queryGroupsState.getValue(); + Map existingStateMap = queryGroupsStateAccessor.getQueryGroupStateMap(); + if (!queryGroupIds.contains("_all")) { + for (String id : queryGroupIds) { + if (!existingStateMap.containsKey(id)) { + throw new ResourceNotFoundException("QueryGroup with id " + id + " does not exist"); + } + } + } + if (existingStateMap != null) { + existingStateMap.forEach((queryGroupId, currentState) -> { + boolean shouldInclude = queryGroupIds.contains("_all") || queryGroupIds.contains(queryGroupId); + if (shouldInclude) { + if (requestedBreached == null || requestedBreached == resourceLimitBreached(queryGroupId, currentState)) { + statsHolderMap.put(queryGroupId, QueryGroupStatsHolder.from(currentState)); + } + } + }); + } + return new QueryGroupStats(statsHolderMap); + } - statsHolderMap.put(queryGroupId, QueryGroupStatsHolder.from(currentState)); + /** + * @return if the QueryGroup breaches any resource limit based on the LastRecordedUsage + */ + public boolean resourceLimitBreached(String id, QueryGroupState currentState) { + QueryGroup queryGroup = clusterService.state().metadata().queryGroups().get(id); + if (queryGroup == null) { + throw new ResourceNotFoundException("QueryGroup with id " + id + " does not exist"); } - return new QueryGroupStats(statsHolderMap); + for (ResourceType resourceType : TRACKED_RESOURCES) { + if (queryGroup.getResourceLimits().containsKey(resourceType)) { + final double threshold = getNormalisedRejectionThreshold(queryGroup.getResourceLimits().get(resourceType), resourceType); + final double lastRecordedUsage = currentState.getResourceState().get(resourceType).getLastRecordedUsage(); + if (threshold < lastRecordedUsage) { + return true; + } + } + } + return false; } /** diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 9d74201de252b..12e0f0ff96f65 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -82,6 +82,10 @@ public int hashCode() { return Objects.hash(stats); } + public Map getStats() { + return stats; + } + /** * This is a stats holder object which will hold the data for a query group at a point in time * the instance will only be created on demand through stats api diff --git a/server/src/main/java/org/opensearch/wlm/stats/WlmStats.java b/server/src/main/java/org/opensearch/wlm/stats/WlmStats.java new file mode 100644 index 0000000000000..3313021bfae52 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/stats/WlmStats.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * This class contains the stats for Workload Management + */ +public class WlmStats extends BaseNodeResponse implements ToXContentObject, Writeable { + private final QueryGroupStats queryGroupStats; + + public WlmStats(DiscoveryNode node, QueryGroupStats queryGroupStats) { + super(node); + this.queryGroupStats = queryGroupStats; + } + + public WlmStats(StreamInput in) throws IOException { + super(in); + queryGroupStats = new QueryGroupStats(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + queryGroupStats.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return queryGroupStats.toXContent(builder, params); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WlmStats that = (WlmStats) o; + return Objects.equals(getQueryGroupStats(), that.getQueryGroupStats()); + } + + @Override + public int hashCode() { + return Objects.hash(queryGroupStats); + } + + public QueryGroupStats getQueryGroupStats() { + return queryGroupStats; + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java new file mode 100644 index 0000000000000..86fbc341691d2 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.wlm; + +import org.opensearch.Version; +import org.opensearch.action.FailedNodeException; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.ResourceType; +import org.opensearch.wlm.stats.QueryGroupStats; +import org.opensearch.wlm.stats.WlmStats; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class WlmStatsResponseTests extends OpenSearchTestCase { + ClusterName clusterName = new ClusterName("test-cluster"); + String testQueryGroupId = "safjgagnaeekg-3r3fads"; + DiscoveryNode node = new DiscoveryNode( + "node-1", + buildNewFakeTransportAddress(), + new HashMap<>(), + Set.of(DiscoveryNodeRole.DATA_ROLE), + Version.CURRENT + ); + Map statsHolderMap = new HashMap<>(); + QueryGroupStats queryGroupStats = new QueryGroupStats( + Map.of( + testQueryGroupId, + new QueryGroupStats.QueryGroupStatsHolder( + 0, + 0, + 1, + 0, + 0, + Map.of( + ResourceType.CPU, + new QueryGroupStats.ResourceStats(0, 0, 0), + ResourceType.MEMORY, + new QueryGroupStats.ResourceStats(0, 0, 0) + ) + ) + ) + ); + WlmStats wlmStats = new WlmStats(node, queryGroupStats); + List wlmStatsList = List.of(wlmStats); + List failedNodeExceptionList = new ArrayList<>(); + + public void testSerializationAndDeserialization() throws IOException { + WlmStatsResponse queryGroupStatsResponse = new WlmStatsResponse(clusterName, wlmStatsList, failedNodeExceptionList); + BytesStreamOutput out = new BytesStreamOutput(); + queryGroupStatsResponse.writeTo(out); + StreamInput in = out.bytes().streamInput(); + WlmStatsResponse deserializedResponse = new WlmStatsResponse(in); + assertEquals(queryGroupStatsResponse.getClusterName(), deserializedResponse.getClusterName()); + assertEquals(queryGroupStatsResponse.getNodes().size(), deserializedResponse.getNodes().size()); + } + + public void testToString() { + WlmStatsResponse queryGroupStatsResponse = new WlmStatsResponse(clusterName, wlmStatsList, failedNodeExceptionList); + String responseString = queryGroupStatsResponse.toString(); + assertEquals( + "{\n" + + " \"node-1\" : {\n" + + " \"query_groups\" : {\n" + + " \"safjgagnaeekg-3r3fads\" : {\n" + + " \"completions\" : 0,\n" + + " \"shard_completions\" : 0,\n" + + " \"rejections\" : 0,\n" + + " \"failures\" : 1,\n" + + " \"total_cancellations\" : 0,\n" + + " \"cpu\" : {\n" + + " \"current_usage\" : 0.0,\n" + + " \"cancellations\" : 0,\n" + + " \"rejections\" : 0\n" + + " },\n" + + " \"memory\" : {\n" + + " \"current_usage\" : 0.0,\n" + + " \"cancellations\" : 0,\n" + + " \"rejections\" : 0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", + responseString + ); + } +} diff --git a/server/src/test/java/org/opensearch/action/support/nodes/TransportWlmStatsActionTests.java b/server/src/test/java/org/opensearch/action/support/nodes/TransportWlmStatsActionTests.java new file mode 100644 index 0000000000000..49d2cc4d23e62 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/support/nodes/TransportWlmStatsActionTests.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.support.nodes; + +import org.opensearch.action.admin.cluster.wlm.TransportWlmStatsAction; +import org.opensearch.action.admin.cluster.wlm.WlmStatsRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.transport.CapturingTransport; +import org.opensearch.wlm.QueryGroupService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; + +public class TransportWlmStatsActionTests extends TransportNodesActionTests { + + /** + * We don't want to send discovery nodes list to each request that is sent across from the coordinator node. + * This behavior is asserted in this test. + */ + public void testWlmStatsActionWithRetentionOfDiscoveryNodesList() { + WlmStatsRequest request = new WlmStatsRequest(); + Map> combinedSentRequest = performWlmStatsAction(request); + + assertNotNull(combinedSentRequest); + combinedSentRequest.forEach((node, capturedRequestList) -> { + assertNotNull(capturedRequestList); + capturedRequestList.forEach(sentRequest -> { assertNull(sentRequest.concreteNodes()); }); + }); + } + + private Map> performWlmStatsAction(WlmStatsRequest request) { + TransportNodesAction action = new TransportWlmStatsAction( + THREAD_POOL, + clusterService, + transportService, + mock(QueryGroupService.class), + new ActionFilters(Collections.emptySet()) + ); + PlainActionFuture listener = new PlainActionFuture<>(); + action.new AsyncAction(null, request, listener).start(); + Map> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear(); + Map> combinedSentRequest = new HashMap<>(); + + capturedRequests.forEach((node, capturedRequestList) -> { + List sentRequestList = new ArrayList<>(); + + capturedRequestList.forEach(preSentRequest -> { + BytesStreamOutput out = new BytesStreamOutput(); + try { + WlmStatsRequest wlmStatsRequestFromCoordinator = (WlmStatsRequest) preSentRequest.request; + wlmStatsRequestFromCoordinator.writeTo(out); + StreamInput in = out.bytes().streamInput(); + WlmStatsRequest wlmStatsRequest = new WlmStatsRequest(in); + sentRequestList.add(wlmStatsRequest); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + combinedSentRequest.put(node, sentRequestList); + }); + + return combinedSentRequest; + } +} diff --git a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java index 1127b50399d24..4d61e89b281f7 100644 --- a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java +++ b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -56,6 +57,10 @@ public void setUp() throws Exception { queryGroupStateMap = new HashMap<>(); testQueryGroupId = "safjgagnakg-3r3fads"; testThreadPool = new TestThreadPool("RejectionTestThreadPool"); + ClusterState mockClusterState = mock(ClusterState.class); + when(mockClusterService.state()).thenReturn(mockClusterState); + Metadata mockMetaData = mock(Metadata.class); + when(mockClusterState.metadata()).thenReturn(mockMetaData); queryGroupService = mock(QueryGroupService.class); sut = new QueryGroupRequestOperationListener(queryGroupService, testThreadPool); } @@ -155,7 +160,9 @@ public void testMultiThreadedValidQueryGroupRequestFailures() { } }); - QueryGroupStats actualStats = queryGroupService.nodeStats(); + HashSet set = new HashSet<>(); + set.add("_all"); + QueryGroupStats actualStats = queryGroupService.nodeStats(set, null); QueryGroupStats expectedStats = new QueryGroupStats( Map.of( @@ -254,11 +261,12 @@ private void assertSuccess( Collections.emptySet(), Collections.emptySet() ); - sut = new QueryGroupRequestOperationListener(queryGroupService, testThreadPool); sut.onRequestFailure(null, null); - QueryGroupStats actualStats = queryGroupService.nodeStats(); + HashSet set = new HashSet<>(); + set.add("_all"); + QueryGroupStats actualStats = queryGroupService.nodeStats(set, null); assertEquals(expectedStats, actualStats); } diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index ac6d19580dacb..bcab0c4d124f5 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -8,17 +8,24 @@ package org.opensearch.wlm.stats; +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.AbstractWireSerializingTestCase; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; import org.opensearch.wlm.ResourceType; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import static java.util.Collections.emptyMap; + public class QueryGroupStatsTests extends AbstractWireSerializingTestCase { public void testToXContent() throws IOException { @@ -72,6 +79,13 @@ protected QueryGroupStats createTestInstance() { ) ) ); + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + VersionUtils.randomCompatibleVersion(random(), Version.CURRENT) + ); return new QueryGroupStats(stats); } } diff --git a/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java new file mode 100644 index 0000000000000..75b8f570b7975 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; +import org.opensearch.wlm.ResourceType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.mockito.Mockito.mock; + +public class WlmStatsTests extends AbstractWireSerializingTestCase { + + public void testToXContent() throws IOException { + final Map stats = new HashMap<>(); + final String queryGroupId = "afakjklaj304041-afaka"; + stats.put( + queryGroupId, + new QueryGroupStats.QueryGroupStatsHolder( + 123456789, + 13, + 2, + 0, + 1213718, + Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(0.3, 13, 2)) + ) + ); + XContentBuilder builder = JsonXContent.contentBuilder(); + QueryGroupStats queryGroupStats = new QueryGroupStats(stats); + WlmStats wlmStats = new WlmStats(mock(DiscoveryNode.class), queryGroupStats); + builder.startObject(); + wlmStats.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals( + "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"shard_completions\":1213718,\"rejections\":13,\"failures\":2,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", + builder.toString() + ); + } + + @Override + protected Writeable.Reader instanceReader() { + return WlmStats::new; + } + + @Override + protected WlmStats createTestInstance() { + DiscoveryNode discoveryNode = new DiscoveryNode( + "node", + OpenSearchTestCase.buildNewFakeTransportAddress(), + emptyMap(), + DiscoveryNodeRole.BUILT_IN_ROLES, + VersionUtils.randomCompatibleVersion(random(), Version.CURRENT) + ); + QueryGroupStatsTests queryGroupStatsTests = new QueryGroupStatsTests(); + return new WlmStats(discoveryNode, queryGroupStatsTests.createTestInstance()); + } +} From b3b8c9b61e90c62c63eca14af0969e2071e79b14 Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Thu, 10 Oct 2024 00:28:23 -0700 Subject: [PATCH 216/357] Add Integration Tests for Workload Management CRUD APIs (#15955) * add IT for workload management CRUD APIs Signed-off-by: Ruirui Zhang * modify based on comments Signed-off-by: Ruirui Zhang * add IT for workload management CRUD APIs Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 1 + plugins/workload-management/build.gradle | 1 + .../rest/WorkloadManagementRestIT.java | 174 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d295c9a199f8..f5622a177ee6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) +- [Workload Management] Add Integration Tests for Workload Management CRUD APIs ([#15955](https://github.com/opensearch-project/OpenSearch/pull/15955)) - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) - [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) - [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777)) diff --git a/plugins/workload-management/build.gradle b/plugins/workload-management/build.gradle index cb14d22ef149f..ad6737bbd24b0 100644 --- a/plugins/workload-management/build.gradle +++ b/plugins/workload-management/build.gradle @@ -10,6 +10,7 @@ */ apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.java-rest-test' apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { diff --git a/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java new file mode 100644 index 0000000000000..7e1d61e57b6f7 --- /dev/null +++ b/plugins/workload-management/src/javaRestTest/java/org/opensearch/rest/WorkloadManagementRestIT.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest; + +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import java.io.IOException; + +public class WorkloadManagementRestIT extends OpenSearchRestTestCase { + + public void testCreate() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + performOperation("DELETE", "_wlm/query_group/analytics", null); + } + + public void testMultipleCreate() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics2", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + + Response response2 = performOperation("PUT", "_wlm/query_group", getCreateJson("users", "soft", 0.2, 0.1)); + assertEquals(response2.getStatusLine().getStatusCode(), 200); + + assertThrows( + ResponseException.class, + () -> performOperation("PUT", "_wlm/query_group", getCreateJson("users2", "soft", 0.41, 0.71)) + ); + performOperation("DELETE", "_wlm/query_group/analytics2", null); + performOperation("DELETE", "_wlm/query_group/users", null); + } + + public void testGet() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics3", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + + Response response2 = performOperation("GET", "_wlm/query_group/analytics3", null); + assertEquals(response2.getStatusLine().getStatusCode(), 200); + String responseBody2 = EntityUtils.toString(response2.getEntity()); + assertTrue(responseBody2.contains("\"name\":\"analytics3\"")); + assertTrue(responseBody2.contains("\"resiliency_mode\":\"enforced\"")); + assertTrue(responseBody2.contains("\"cpu\":0.4")); + assertTrue(responseBody2.contains("\"memory\":0.2")); + + assertThrows(ResponseException.class, () -> performOperation("GET", "_wlm/query_group/analytics97", null)); + performOperation("DELETE", "_wlm/query_group/analytics3", null); + } + + public void testDelete() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics4", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + + Response response2 = performOperation("DELETE", "_wlm/query_group/analytics4", null); + assertEquals(response2.getStatusLine().getStatusCode(), 200); + assertTrue(EntityUtils.toString(response2.getEntity()).contains("\"acknowledged\":true")); + + assertThrows(ResponseException.class, () -> performOperation("DELETE", "_wlm/query_group/analytics99", null)); + } + + public void testUpdate() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics5", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + + Response response2 = performOperation("PUT", "_wlm/query_group/analytics5", getUpdateJson("monitor", 0.41, 0.21)); + assertEquals(response2.getStatusLine().getStatusCode(), 200); + String responseBody2 = EntityUtils.toString(response2.getEntity()); + assertTrue(responseBody2.contains("\"name\":\"analytics5\"")); + assertTrue(responseBody2.contains("\"resiliency_mode\":\"monitor\"")); + assertTrue(responseBody2.contains("\"cpu\":0.41")); + assertTrue(responseBody2.contains("\"memory\":0.21")); + + String json = "{\n" + + " \"resource_limits\": {\n" + + " \"cpu\" : 1.1,\n" + + " \"memory\" : -0.1\n" + + " }\n" + + "}'"; + assertThrows(ResponseException.class, () -> performOperation("PUT", "_wlm/query_group/analytics5", json)); + assertThrows( + ResponseException.class, + () -> performOperation("PUT", "_wlm/query_group/analytics98", getUpdateJson("monitor", 0.43, 0.23)) + ); + performOperation("DELETE", "_wlm/query_group/analytics5", null); + } + + public void testCRUD() throws Exception { + Response response = performOperation("PUT", "_wlm/query_group", getCreateJson("analytics6", "enforced", 0.4, 0.2)); + assertEquals(response.getStatusLine().getStatusCode(), 200); + + Response response1 = performOperation("PUT", "_wlm/query_group/analytics6", getUpdateJson("monitor", 0.41, 0.21)); + assertEquals(response1.getStatusLine().getStatusCode(), 200); + + Response response2 = performOperation("GET", "_wlm/query_group/analytics6", null); + assertEquals(response2.getStatusLine().getStatusCode(), 200); + String responseBody2 = EntityUtils.toString(response2.getEntity()); + assertTrue(responseBody2.contains("\"name\":\"analytics6\"")); + assertTrue(responseBody2.contains("\"resiliency_mode\":\"monitor\"")); + assertTrue(responseBody2.contains("\"cpu\":0.41")); + assertTrue(responseBody2.contains("\"memory\":0.21")); + + assertThrows( + ResponseException.class, + () -> performOperation("PUT", "_wlm/query_group", getCreateJson("users3", "monitor", 0.6, 0.8)) + ); + + Response response4 = performOperation("PUT", "_wlm/query_group", getCreateJson("users3", "monitor", 0.59, 0.79)); + assertEquals(response4.getStatusLine().getStatusCode(), 200); + + Response response5 = performOperation("DELETE", "_wlm/query_group/analytics6", null); + assertEquals(response5.getStatusLine().getStatusCode(), 200); + String responseBody5 = EntityUtils.toString(response5.getEntity()); + assertTrue(responseBody5.contains("\"acknowledged\":true")); + + Response response6 = performOperation("GET", "_wlm/query_group", null); + assertEquals(response6.getStatusLine().getStatusCode(), 200); + String responseBody6 = EntityUtils.toString(response6.getEntity()); + assertTrue(responseBody6.contains("\"query_groups\"")); + assertTrue(responseBody6.contains("\"users3\"")); + assertFalse(responseBody6.contains("\"analytics6\"")); + performOperation("DELETE", "_wlm/query_group/users3", null); + } + + static String getCreateJson(String name, String resiliencyMode, double cpu, double memory) { + return "{\n" + + " \"name\": \"" + + name + + "\",\n" + + " \"resiliency_mode\": \"" + + resiliencyMode + + "\",\n" + + " \"resource_limits\": {\n" + + " \"cpu\" : " + + cpu + + ",\n" + + " \"memory\" : " + + memory + + "\n" + + " }\n" + + "}"; + } + + static String getUpdateJson(String resiliencyMode, double cpu, double memory) { + return "{\n" + + " \"resiliency_mode\": \"" + + resiliencyMode + + "\",\n" + + " \"resource_limits\": {\n" + + " \"cpu\" : " + + cpu + + ",\n" + + " \"memory\" : " + + memory + + "\n" + + " }\n" + + "}"; + } + + Response performOperation(String method, String uriPath, String json) throws IOException { + Request request = new Request(method, uriPath); + if (json != null) { + request.setJsonEntity(json); + } + return client().performRequest(request); + } +} From 691f7256fb5eaf2e285b13d6b10524d29e427ee9 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:09:30 +0530 Subject: [PATCH 217/357] Reconfigure remote state thread pool count (#16245) * Reconfigure remote state thread pool count Signed-off-by: Sooraj Sinha --- .../repositories/s3/S3RepositoryPlugin.java | 2 +- .../s3/S3RepositoryPluginTests.java | 81 +++++++++++++++++++ .../org/opensearch/threadpool/ThreadPool.java | 7 +- .../threadpool/ScalingThreadPoolTests.java | 2 +- 4 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 110d91bfbd822..1048ec784ec4e 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -188,7 +188,7 @@ private static int allocatedProcessors(Settings settings) { } private static int urgentPoolCount(Settings settings) { - return boundedBy((allocatedProcessors(settings) + 7) / 8, 1, 2); + return boundedBy((allocatedProcessors(settings) + 1) / 2, 1, 2); } private static int priorityPoolCount(Settings settings) { diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java new file mode 100644 index 0000000000000..9ac1564c807c3 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.SizeUnit; +import org.opensearch.common.unit.SizeValue; +import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.threadpool.ThreadPool.ThreadPoolType; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.Executor; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public class S3RepositoryPluginTests extends OpenSearchTestCase { + + private static final String URGENT_FUTURE_COMPLETION = "urgent_future_completion"; + + public void testGetExecutorBuilders() throws IOException { + final int processors = randomIntBetween(1, 64); + Settings settings = Settings.builder().put("node.name", "test").put("node.processors", processors).build(); + Path configPath = createTempDir(); + ThreadPool threadPool = null; + try (S3RepositoryPlugin plugin = new S3RepositoryPlugin(settings, configPath)) { + List> executorBuilders = plugin.getExecutorBuilders(settings); + assertNotNull(executorBuilders); + assertFalse(executorBuilders.isEmpty()); + threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0])); + final Executor executor = threadPool.executor(URGENT_FUTURE_COMPLETION); + assertNotNull(executor); + assertThat(executor, instanceOf(OpenSearchThreadPoolExecutor.class)); + final OpenSearchThreadPoolExecutor openSearchThreadPoolExecutor = (OpenSearchThreadPoolExecutor) executor; + final ThreadPool.Info info = threadPool.info(URGENT_FUTURE_COMPLETION); + int size = boundedBy((processors + 1) / 2, 1, 2); + assertThat(info.getName(), equalTo(URGENT_FUTURE_COMPLETION)); + assertThat(info.getThreadPoolType(), equalTo(ThreadPoolType.FIXED)); + assertThat(info.getQueueSize(), notNullValue()); + assertThat(info.getQueueSize(), equalTo(new SizeValue(10, SizeUnit.KILO))); + assertThat(openSearchThreadPoolExecutor.getQueue().remainingCapacity(), equalTo(10_000)); + + assertThat(info.getMin(), equalTo(size)); + assertThat(openSearchThreadPoolExecutor.getCorePoolSize(), equalTo(size)); + assertThat(info.getMax(), equalTo(size)); + assertThat(openSearchThreadPoolExecutor.getMaximumPoolSize(), equalTo(size)); + + final int availableProcessors = Runtime.getRuntime().availableProcessors(); + if (processors > availableProcessors) { + assertWarnings( + "setting [node.processors] to value [" + + processors + + "] which is more than available processors [" + + availableProcessors + + "] is deprecated" + ); + } + } finally { + if (threadPool != null) { + terminate(threadPool); + } + } + } + + private static int boundedBy(int value, int min, int max) { + return Math.min(max, Math.max(min, value)); + } + +} diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index d795fd252b7fc..e0b15e54f6e2e 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -293,12 +293,7 @@ public ThreadPool( ); builders.put( Names.REMOTE_STATE_READ, - new ScalingExecutorBuilder( - Names.REMOTE_STATE_READ, - 1, - twiceAllocatedProcessors(allocatedProcessors), - TimeValue.timeValueMinutes(5) - ) + new ScalingExecutorBuilder(Names.REMOTE_STATE_READ, 1, boundedBy(4 * allocatedProcessors, 4, 32), TimeValue.timeValueMinutes(5)) ); builders.put( Names.INDEX_SEARCHER, diff --git a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java index 4f59f9688fb7e..b4726bab50198 100644 --- a/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ScalingThreadPoolTests.java @@ -156,7 +156,7 @@ private int expectedSize(final String threadPoolName, final int numberOfProcesso sizes.put(ThreadPool.Names.REMOTE_PURGE, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_REFRESH_RETRY, ThreadPool::halfAllocatedProcessors); sizes.put(ThreadPool.Names.REMOTE_RECOVERY, ThreadPool::twiceAllocatedProcessors); - sizes.put(ThreadPool.Names.REMOTE_STATE_READ, ThreadPool::twiceAllocatedProcessors); + sizes.put(ThreadPool.Names.REMOTE_STATE_READ, n -> ThreadPool.boundedBy(4 * n, 4, 32)); return sizes.get(threadPoolName).apply(numberOfProcessors); } From 942765e148845b0d563ca9863d3b418984814db6 Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Thu, 10 Oct 2024 17:19:59 +0530 Subject: [PATCH 218/357] Add snapshot_path prefix to snapshot shards path file on S3 (#16267) Signed-off-by: Ashish Singh --- .../blobstore/BlobStoreRepository.java | 16 ++++++++++------ .../opensearch/snapshots/SnapshotShardPaths.java | 12 ++++++++++-- .../blobstore/BlobStoreRepositoryTests.java | 13 +++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 8482146530860..243d0021fac2e 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -199,6 +199,7 @@ import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS; +import static org.opensearch.snapshots.SnapshotShardPaths.getIndexId; /** * BlobStore - based implementation of Snapshot Repository @@ -2293,7 +2294,10 @@ private void remoteTranslogCleanupAsync( * @return List of matching shard paths */ private List findMatchingShardPaths(String indexId, Map snapshotShardPaths) { - return snapshotShardPaths.keySet().stream().filter(s -> s.startsWith(indexId)).collect(Collectors.toList()); + return snapshotShardPaths.keySet() + .stream() + .filter(s -> (s.startsWith(indexId) || s.startsWith(SnapshotShardPaths.FILE_PREFIX + indexId))) + .collect(Collectors.toList()); } /** @@ -2546,11 +2550,11 @@ public void finalizeSnapshot( */ private void cleanupRedundantSnapshotShardPaths(Set updatedShardPathsIndexIds) { Set updatedIndexIds = updatedShardPathsIndexIds.stream() - .map(s -> s.split("\\" + SnapshotShardPaths.DELIMITER)[0]) + .map(s -> getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0])) .collect(Collectors.toSet()); Set indexIdShardPaths = getSnapshotShardPaths().keySet(); List staleShardPaths = indexIdShardPaths.stream().filter(s -> updatedShardPathsIndexIds.contains(s) == false).filter(s -> { - String indexId = s.split("\\" + SnapshotShardPaths.DELIMITER)[0]; + String indexId = getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0]); return updatedIndexIds.contains(indexId); }).collect(Collectors.toList()); try { @@ -2595,7 +2599,7 @@ String writeIndexShardPaths(IndexId indexId, SnapshotId snapshotId, int shardCou List paths = getShardPaths(indexId, shardCount); int pathType = indexId.getShardPathType(); int pathHashAlgorithm = FNV_1A_COMPOSITE_1.getCode(); - String blobName = String.join( + String name = String.join( SnapshotShardPaths.DELIMITER, indexId.getId(), indexId.getName(), @@ -2611,9 +2615,9 @@ String writeIndexShardPaths(IndexId indexId, SnapshotId snapshotId, int shardCou PathType.fromCode(pathType), PathHashAlgorithm.fromCode(pathHashAlgorithm) ); - SNAPSHOT_SHARD_PATHS_FORMAT.write(shardPaths, snapshotShardPathBlobContainer(), blobName); + SNAPSHOT_SHARD_PATHS_FORMAT.write(shardPaths, snapshotShardPathBlobContainer(), name); logShardPathsOperationSuccess(indexId, snapshotId); - return blobName; + return SnapshotShardPaths.FILE_PREFIX + name; } catch (IOException e) { logShardPathsOperationWarning(indexId, snapshotId, e); } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java index 88af14e2232f9..878c2baba4ce9 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java @@ -29,7 +29,8 @@ public class SnapshotShardPaths implements ToXContent { public static final String DELIMITER = "."; - public static final String FILE_NAME_FORMAT = "%s"; + public static final String FILE_PREFIX = "snapshot_path_"; + public static final String FILE_NAME_FORMAT = FILE_PREFIX + "%s"; private static final String PATHS_FIELD = "paths"; private static final String INDEX_ID_FIELD = "indexId"; @@ -101,7 +102,7 @@ public static ShardInfo parseShardPath(String shardPath) { throw new IllegalArgumentException("Invalid shard path format: " + shardPath); } try { - IndexId indexId = new IndexId(parts[1], parts[0], Integer.parseInt(parts[3])); + IndexId indexId = new IndexId(parts[1], getIndexId(parts[0]), Integer.parseInt(parts[3])); int shardCount = Integer.parseInt(parts[2]); return new ShardInfo(indexId, shardCount); } catch (NumberFormatException e) { @@ -109,6 +110,13 @@ public static ShardInfo parseShardPath(String shardPath) { } } + public static String getIndexId(String indexIdField) { + if (indexIdField.startsWith(FILE_PREFIX)) { + return indexIdField.substring(FILE_PREFIX.length()); + } + return indexIdField; + } + /** * Represents parsed information from a shard path. * This class encapsulates the index ID and shard count extracted from a shard path string. diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index d9622aae4c378..aa10b7dc18381 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -382,6 +382,7 @@ public void testParseShardPath() { IndexId indexId = repoData.getIndices().values().iterator().next(); int shardCount = repoData.shardGenerations().getGens(indexId).size(); + // Version 2.17 has file name starting with indexId String shardPath = String.join( SnapshotShardPaths.DELIMITER, indexId.getId(), @@ -391,7 +392,19 @@ public void testParseShardPath() { "1" ); ShardInfo shardInfo = SnapshotShardPaths.parseShardPath(shardPath); + assertEquals(shardInfo.getIndexId(), indexId); + assertEquals(shardInfo.getShardCount(), shardCount); + // Version 2.17 has file name starting with snapshot_path_ + shardPath = String.join( + SnapshotShardPaths.DELIMITER, + SnapshotShardPaths.FILE_PREFIX + indexId.getId(), + indexId.getName(), + String.valueOf(shardCount), + String.valueOf(indexId.getShardPathType()), + "1" + ); + shardInfo = SnapshotShardPaths.parseShardPath(shardPath); assertEquals(shardInfo.getIndexId(), indexId); assertEquals(shardInfo.getShardCount(), shardCount); } From 1e7f6df79c7845ba04ecc4a05979db27965342c7 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 10 Oct 2024 19:51:26 -0400 Subject: [PATCH 219/357] Support JDK-23 (build time and runtime) (#16257) Signed-off-by: Andriy Redko --- .github/workflows/precommit.yml | 2 +- buildSrc/build.gradle | 2 +- .../precommit/ThirdPartyAuditPrecommitPlugin.java | 2 +- .../opensearch/gradle/test/DistroTestPlugin.java | 4 ++-- buildSrc/version.properties | 6 +++--- gradle/code-coverage.gradle | 2 +- .../org/opensearch/grok/GrokCaptureConfig.java | 14 +++++++------- .../discovery/ec2/Ec2DiscoveryTests.java | 1 + .../opensearch/discovery/ec2/Ec2RetriesTests.java | 1 + plugins/repository-hdfs/build.gradle | 4 ++++ .../lucene/search/MultiPhrasePrefixQuery.java | 2 -- .../org/opensearch/index/search/MatchQuery.java | 3 --- .../replication/SegmentReplicationSource.java | 4 ++-- .../AbstractBlobContainerRetriesTestCase.java | 4 +--- 14 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 7c65df1f677a5..572f6c981a052 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 11, 17, 21, 23 ] os: [ubuntu-latest, windows-latest, macos-latest, macos-13] steps: - uses: actions/checkout@v4 diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b984ef3800490..aaa1d8bdef85c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -114,7 +114,7 @@ dependencies { api 'com.github.johnrengelman:shadow:8.1.1' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" - api 'de.thetaphi:forbiddenapis:3.6' + api 'de.thetaphi:forbiddenapis:3.8' api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.6' diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java index d83f1b01ee043..988b8b749ee64 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public TaskProvider createTask(Project project) { project.getPlugins().apply(CompileOnlyResolvePlugin.class); project.getConfigurations().create("forbiddenApisCliJar"); - project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.5.1"); + project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.8"); Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME); if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 8d5ce9143cbac..9365f1c732229 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -77,9 +77,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "21.0.4+7"; + private static final String SYSTEM_JDK_VERSION = "23+37"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "21.0.4+7"; + private static final String GRADLE_JDK_VERSION = "23+37"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 9affa9b9a45f1..4bb6f3af7fbe4 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -2,7 +2,7 @@ opensearch = 3.0.0 lucene = 9.12.0 bundled_jdk_vendor = adoptium -bundled_jdk = 21.0.4+7 +bundled_jdk = 23+37 # optional dependencies spatial4j = 0.7 @@ -60,9 +60,9 @@ bouncycastle=1.78 randomizedrunner = 2.7.1 junit = 4.13.2 hamcrest = 2.1 -mockito = 5.12.0 +mockito = 5.14.1 objenesis = 3.2 -bytebuddy = 1.14.9 +bytebuddy = 1.15.4 # benchmark dependencies jmh = 1.35 diff --git a/gradle/code-coverage.gradle b/gradle/code-coverage.gradle index 3ca6b1fe84ea7..eb27dd1a76634 100644 --- a/gradle/code-coverage.gradle +++ b/gradle/code-coverage.gradle @@ -19,7 +19,7 @@ repositories { allprojects { plugins.withId('jacoco') { - jacoco.toolVersion = '0.8.10' + jacoco.toolVersion = '0.8.12' } } diff --git a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureConfig.java b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureConfig.java index 6d1985f2165cd..6fabad3a7d5bd 100644 --- a/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureConfig.java +++ b/libs/grok/src/main/java/org/opensearch/grok/GrokCaptureConfig.java @@ -114,8 +114,8 @@ public GrokCaptureExtracter forBoolean(Function, GrokCaptureEx /** * Build an extract that has access to the "native" type of the extracter * match. This means that patterns like {@code %{NUMBER:bytes:float}} has - * access to an actual {@link float}. Extracters returned from this method - * should be stateless stateless and can be reused. Pathological implementations + * access to an actual float. Extracters returned from this method + * should be stateless and can be reused. Pathological implementations * of the {@code map} parameter could violate this, but the caller should * take care to stay sane. *

    @@ -144,27 +144,27 @@ public interface NativeExtracterMap { T forString(Function, GrokCaptureExtracter> buildExtracter); /** - * Called when the native type is an {@link int}. + * Called when the native type is an int. */ T forInt(Function buildExtracter); /** - * Called when the native type is an {@link long}. + * Called when the native type is an long. */ T forLong(Function buildExtracter); /** - * Called when the native type is an {@link float}. + * Called when the native type is an float. */ T forFloat(Function buildExtracter); /** - * Called when the native type is an {@link double}. + * Called when the native type is an double. */ T forDouble(Function buildExtracter); /** - * Called when the native type is an {@link boolean}. + * Called when the native type is an boolean. */ T forBoolean(Function, GrokCaptureExtracter> buildExtracter); } diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java index 02e1ff40f7ed6..7a56cfdb6278e 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java @@ -175,6 +175,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java index ce097667f9c4b..194a262c6ed7f 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java @@ -125,6 +125,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException { exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 15c4a1647fd38..faa9b2bfff84d 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -146,6 +146,10 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', } final List miniHDFSArgs = [] + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_23) { + miniHDFSArgs.add('-Djava.security.manager=allow') + } + // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}"); diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java index cc0468efb243e..5563511aa8d45 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -90,8 +90,6 @@ public void setMaxExpansions(int maxExpansions) { /** * Sets the phrase slop for this query. - * - * @see org.apache.lucene.search.PhraseQuery.Builder#getSlop() */ public int getSlop() { return slop; diff --git a/server/src/main/java/org/opensearch/index/search/MatchQuery.java b/server/src/main/java/org/opensearch/index/search/MatchQuery.java index ec6755ea25703..86ea799ab311d 100644 --- a/server/src/main/java/org/opensearch/index/search/MatchQuery.java +++ b/server/src/main/java/org/opensearch/index/search/MatchQuery.java @@ -227,9 +227,6 @@ public void setOccur(BooleanClause.Occur occur) { this.occur = occur; } - /** - * @deprecated See {@link MatchQueryBuilder#setCommonTermsCutoff(Float)} for more details - */ @Deprecated public void setCommonTermsCutoff(Float cutoff) { this.commonTermsCutoff = cutoff; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java index 24f0cb15ddb25..5341f9507bef4 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSource.java @@ -32,7 +32,7 @@ public interface SegmentReplicationSource { /** * Get Metadata for a ReplicationCheckpoint. * - * @param replicationId {@link long} - ID of the replication event. + * @param replicationId long - ID of the replication event. * @param checkpoint {@link ReplicationCheckpoint} Checkpoint to fetch metadata for. * @param listener {@link ActionListener} listener that completes with a {@link CheckpointInfoResponse}. */ @@ -41,7 +41,7 @@ public interface SegmentReplicationSource { /** * Fetch the requested segment files. Passes a listener that completes when files are stored locally. * - * @param replicationId {@link long} - ID of the replication event. + * @param replicationId long - ID of the replication event. * @param checkpoint {@link ReplicationCheckpoint} Checkpoint to fetch metadata for. * @param filesToFetch {@link List} List of files to fetch. * @param indexShard {@link IndexShard} Reference to the IndexShard. diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 74c75ea05b1f3..e3d75cbb69638 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -404,8 +404,6 @@ protected void sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws if (bytesToSend > 0) { exchange.getResponseBody().write(bytes, rangeStart, bytesToSend); } - if (randomBoolean()) { - exchange.getResponseBody().flush(); - } + exchange.getResponseBody().flush(); } } From 34b794b93b94d0329fcd6eb8ec2f70fa782729ad Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Thu, 10 Oct 2024 23:25:11 -0700 Subject: [PATCH 220/357] Making wlm stats output easier to understand (#16278) * fix wlm stats output Signed-off-by: Kaushal Kumar * rename wlm stats vars Signed-off-by: Kaushal Kumar * fix ut failure Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kaushal Kumar --- .../org/opensearch/wlm/QueryGroupService.java | 7 +--- .../opensearch/wlm/stats/QueryGroupState.java | 19 ++------- .../opensearch/wlm/stats/QueryGroupStats.java | 39 ++++++++----------- .../cluster/wlm/WlmStatsResponseTests.java | 7 +--- .../wlm/QueryGroupServiceTests.java | 4 +- ...eryGroupRequestOperationListenerTests.java | 6 --- .../wlm/stats/QueryGroupStateTests.java | 10 +---- .../wlm/stats/QueryGroupStatsTests.java | 4 +- .../opensearch/wlm/stats/WlmStatsTests.java | 3 +- 9 files changed, 28 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java index 30d39686931be..cb0be5597766a 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.search.SearchShardTask; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; import org.opensearch.cluster.metadata.Metadata; @@ -354,10 +353,6 @@ public void onTaskCompleted(Task task) { queryGroupId = QueryGroupTask.DEFAULT_QUERY_GROUP_ID_SUPPLIER.get(); } - if (task instanceof SearchShardTask) { - queryGroupsStateAccessor.getQueryGroupState(queryGroupId).shardCompletions.inc(); - } else { - queryGroupsStateAccessor.getQueryGroupState(queryGroupId).completions.inc(); - } + queryGroupsStateAccessor.getQueryGroupState(queryGroupId).totalCompletions.inc(); } } diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java index cbc7046a79464..a082ed159e829 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -21,12 +21,7 @@ public class QueryGroupState { /** * co-ordinator level completions at the query group level, this is a cumulative counter since the Opensearch start time */ - public final CounterMetric completions = new CounterMetric(); - - /** - * shard level completions at the query group level, this is a cumulative counter since the Opensearch start time - */ - public final CounterMetric shardCompletions = new CounterMetric(); + public final CounterMetric totalCompletions = new CounterMetric(); /** * rejections at the query group level, this is a cumulative counter since the OpenSearch start time @@ -61,16 +56,8 @@ public QueryGroupState() { * * @return co-ordinator completions in the query group */ - public long getCompletions() { - return completions.count(); - } - - /** - * - * @return shard completions in the query group - */ - public long getShardCompletions() { - return shardCompletions.count(); + public long getTotalCompletions() { + return totalCompletions.count(); } /** diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 12e0f0ff96f65..42ce3ac0019db 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -91,16 +91,14 @@ public Map getStats() { * the instance will only be created on demand through stats api */ public static class QueryGroupStatsHolder implements ToXContentObject, Writeable { - public static final String COMPLETIONS = "completions"; - public static final String REJECTIONS = "rejections"; + public static final String COMPLETIONS = "total_completions"; + public static final String REJECTIONS = "total_rejections"; public static final String TOTAL_CANCELLATIONS = "total_cancellations"; public static final String FAILURES = "failures"; - public static final String SHARD_COMPLETIONS = "shard_completions"; private long completions; - private long shardCompletions; private long rejections; private long failures; - private long totalCancellations; + private long cancellations; private Map resourceStats; // this is needed to support the factory method @@ -110,15 +108,13 @@ public QueryGroupStatsHolder( long completions, long rejections, long failures, - long totalCancellations, - long shardCompletions, + long cancellations, Map resourceStats ) { this.completions = completions; this.rejections = rejections; this.failures = failures; - this.shardCompletions = shardCompletions; - this.totalCancellations = totalCancellations; + this.cancellations = cancellations; this.resourceStats = resourceStats; } @@ -126,8 +122,7 @@ public QueryGroupStatsHolder(StreamInput in) throws IOException { this.completions = in.readVLong(); this.rejections = in.readVLong(); this.failures = in.readVLong(); - this.totalCancellations = in.readVLong(); - this.shardCompletions = in.readVLong(); + this.cancellations = in.readVLong(); this.resourceStats = in.readMap((i) -> ResourceType.fromName(i.readString()), ResourceStats::new); } @@ -145,11 +140,10 @@ public static QueryGroupStatsHolder from(QueryGroupState queryGroupState) { resourceStatsMap.put(resourceTypeStateEntry.getKey(), ResourceStats.from(resourceTypeStateEntry.getValue())); } - statsHolder.completions = queryGroupState.getCompletions(); + statsHolder.completions = queryGroupState.getTotalCompletions(); statsHolder.rejections = queryGroupState.getTotalRejections(); statsHolder.failures = queryGroupState.getFailures(); - statsHolder.totalCancellations = queryGroupState.getTotalCancellations(); - statsHolder.shardCompletions = queryGroupState.getShardCompletions(); + statsHolder.cancellations = queryGroupState.getTotalCancellations(); statsHolder.resourceStats = resourceStatsMap; return statsHolder; } @@ -164,8 +158,7 @@ public static void writeTo(StreamOutput out, QueryGroupStatsHolder statsHolder) out.writeVLong(statsHolder.completions); out.writeVLong(statsHolder.rejections); out.writeVLong(statsHolder.failures); - out.writeVLong(statsHolder.totalCancellations); - out.writeVLong(statsHolder.shardCompletions); + out.writeVLong(statsHolder.cancellations); out.writeMap(statsHolder.resourceStats, (o, val) -> o.writeString(val.getName()), ResourceStats::writeTo); } @@ -177,10 +170,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(COMPLETIONS, completions); - builder.field(SHARD_COMPLETIONS, shardCompletions); + // builder.field(SHARD_COMPLETIONS, shardCompletions); builder.field(REJECTIONS, rejections); - builder.field(FAILURES, failures); - builder.field(TOTAL_CANCELLATIONS, totalCancellations); + // builder.field(FAILURES, failures); + builder.field(TOTAL_CANCELLATIONS, cancellations); for (ResourceType resourceType : ResourceType.getSortedValues()) { ResourceStats resourceStats1 = resourceStats.get(resourceType); @@ -199,15 +192,14 @@ public boolean equals(Object o) { QueryGroupStatsHolder that = (QueryGroupStatsHolder) o; return completions == that.completions && rejections == that.rejections - && shardCompletions == that.shardCompletions && Objects.equals(resourceStats, that.resourceStats) && failures == that.failures - && totalCancellations == that.totalCancellations; + && cancellations == that.cancellations; } @Override public int hashCode() { - return Objects.hash(completions, shardCompletions, rejections, totalCancellations, failures, resourceStats); + return Objects.hash(completions, rejections, cancellations, failures, resourceStats); } } @@ -217,6 +209,7 @@ public int hashCode() { public static class ResourceStats implements ToXContentObject, Writeable { public static final String CURRENT_USAGE = "current_usage"; public static final String CANCELLATIONS = "cancellations"; + public static final String REJECTIONS = "rejections"; public static final double PRECISION = 1e-9; private final double currentUsage; private final long cancellations; @@ -268,7 +261,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(CURRENT_USAGE, currentUsage); builder.field(CANCELLATIONS, cancellations); - builder.field(QueryGroupStatsHolder.REJECTIONS, rejections); + builder.field(REJECTIONS, rejections); return builder; } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java index 86fbc341691d2..01dc033568a95 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/wlm/WlmStatsResponseTests.java @@ -46,7 +46,6 @@ public class WlmStatsResponseTests extends OpenSearchTestCase { 0, 1, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -78,10 +77,8 @@ public void testToString() { + " \"node-1\" : {\n" + " \"query_groups\" : {\n" + " \"safjgagnaeekg-3r3fads\" : {\n" - + " \"completions\" : 0,\n" - + " \"shard_completions\" : 0,\n" - + " \"rejections\" : 0,\n" - + " \"failures\" : 1,\n" + + " \"total_completions\" : 0,\n" + + " \"total_rejections\" : 0,\n" + " \"total_cancellations\" : 0,\n" + " \"cpu\" : {\n" + " \"current_usage\" : 0.0,\n" diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java index c5cf0dac4f807..45428865259c3 100644 --- a/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java @@ -401,14 +401,14 @@ public void testOnTaskCompleted() { ((QueryGroupTask) task).setQueryGroupId(mockThreadPool.getThreadContext()); queryGroupService.onTaskCompleted(task); - assertEquals(1, queryGroupState.completions.count()); + assertEquals(1, queryGroupState.totalCompletions.count()); // test non QueryGroupTask task = new Task(1, "simple", "test", "mock task", null, null); queryGroupService.onTaskCompleted(task); // It should still be 1 - assertEquals(1, queryGroupState.completions.count()); + assertEquals(1, queryGroupState.totalCompletions.count()); mockThreadPool.shutdown(); } diff --git a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java index 4d61e89b281f7..016588acf1e24 100644 --- a/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java +++ b/server/src/test/java/org/opensearch/wlm/listeners/QueryGroupRequestOperationListenerTests.java @@ -95,7 +95,6 @@ public void testValidQueryGroupRequestFailure() throws IOException { 0, 1, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -109,7 +108,6 @@ public void testValidQueryGroupRequestFailure() throws IOException { 0, 0, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -172,7 +170,6 @@ public void testMultiThreadedValidQueryGroupRequestFailures() { 0, ITERATIONS, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -186,7 +183,6 @@ public void testMultiThreadedValidQueryGroupRequestFailures() { 0, 0, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -209,7 +205,6 @@ public void testInvalidQueryGroupFailure() throws IOException { 0, 0, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), @@ -223,7 +218,6 @@ public void testInvalidQueryGroupFailure() throws IOException { 0, 1, 0, - 0, Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats(0, 0, 0), diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java index 566c4261d6878..c0dfa06a0fba1 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -23,13 +23,7 @@ public void testRandomQueryGroupsStateUpdates() { for (int i = 0; i < 25; i++) { if (i % 5 == 0) { - updaterThreads.add(new Thread(() -> { - if (randomBoolean()) { - queryGroupState.completions.inc(); - } else { - queryGroupState.shardCompletions.inc(); - } - })); + updaterThreads.add(new Thread(() -> { queryGroupState.totalCompletions.inc(); })); } else if (i % 5 == 1) { updaterThreads.add(new Thread(() -> { queryGroupState.totalRejections.inc(); @@ -63,7 +57,7 @@ public void testRandomQueryGroupsStateUpdates() { } }); - assertEquals(5, queryGroupState.getCompletions() + queryGroupState.getShardCompletions()); + assertEquals(5, queryGroupState.getTotalCompletions()); assertEquals(5, queryGroupState.getTotalRejections()); final long sumOfRejectionsDueToResourceTypes = queryGroupState.getResourceState().get(ResourceType.CPU).rejections.count() diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index bcab0c4d124f5..6fc4d178e54bc 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -38,7 +38,6 @@ public void testToXContent() throws IOException { 13, 2, 0, - 1213718, Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(0.3, 13, 2)) ) ); @@ -48,7 +47,7 @@ public void testToXContent() throws IOException { queryGroupStats.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); assertEquals( - "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"shard_completions\":1213718,\"rejections\":13,\"failures\":2,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", + "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"total_completions\":123456789,\"total_rejections\":13,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", builder.toString() ); } @@ -68,7 +67,6 @@ protected QueryGroupStats createTestInstance() { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), - randomNonNegativeLong(), Map.of( ResourceType.CPU, new QueryGroupStats.ResourceStats( diff --git a/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java index 75b8f570b7975..6910ca7f9937c 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/WlmStatsTests.java @@ -39,7 +39,6 @@ public void testToXContent() throws IOException { 13, 2, 0, - 1213718, Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(0.3, 13, 2)) ) ); @@ -50,7 +49,7 @@ public void testToXContent() throws IOException { wlmStats.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); assertEquals( - "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"shard_completions\":1213718,\"rejections\":13,\"failures\":2,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", + "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"total_completions\":123456789,\"total_rejections\":13,\"total_cancellations\":0,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", builder.toString() ); } From d6ea8eb56077de73e01d8a8a4eb11d3d4a52dea8 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 11 Oct 2024 16:10:00 +0530 Subject: [PATCH 221/357] Change remote store restore cluster state priority to URGENT (#16281) Signed-off-by: Sachin Kale --- .../opensearch/index/recovery/RemoteStoreRestoreService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java index 03d841d13b7f7..73a79a54ca588 100644 --- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java +++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java @@ -27,6 +27,7 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.common.Priority; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; @@ -95,7 +96,7 @@ public RemoteStoreRestoreService( * @param listener restore listener */ public void restore(RestoreRemoteStoreRequest request, final ActionListener listener) { - clusterService.submitStateUpdateTask("restore[remote_store]", new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("restore[remote_store]", new ClusterStateUpdateTask(Priority.URGENT) { String restoreUUID; RestoreInfo restoreInfo = null; From b3459fdf0f07b3dfe1d8375fbc1db50840f7a8fe Mon Sep 17 00:00:00 2001 From: Liyun Xiu Date: Sat, 12 Oct 2024 02:43:36 +0800 Subject: [PATCH 222/357] Update setting API honors cluster's replica setting as default #14810 (#14948) * Update setting API honors cluster's replica setting as default #14810 Signed-off-by: Liyun Xiu * Update changelog & add one more test case Signed-off-by: Liyun Xiu --------- Signed-off-by: Liyun Xiu Signed-off-by: Daniel Widdis Co-authored-by: Daniel Widdis --- CHANGELOG.md | 1 + .../indices/settings/UpdateSettingsIT.java | 63 +++++++++++++++++++ .../MetadataUpdateSettingsService.java | 18 +++--- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5622a177ee6c..2a32d2503bbef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) +- Fix update settings with null replica not honoring cluster setting bug ([#14948](https://github.com/opensearch-project/OpenSearch/pull/14948)) - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java index 475d0a154a98b..5e90ee3185618 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java @@ -895,6 +895,69 @@ private void runTestDefaultNumberOfReplicasTest(final boolean closeIndex) { assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(response.getIndexToSettings().get("test")), equalTo(1)); } + public void testNullReplicaUpdate() { + internalCluster().ensureAtLeastNumDataNodes(2); + + // cluster setting + String defaultNumberOfReplica = "3"; + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put("cluster.default_number_of_replicas", defaultNumberOfReplica)) + .get() + ); + // create index with number of replicas will ignore default value + assertAcked( + client().admin() + .indices() + .prepareCreate("test") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "2")) + ); + + String numberOfReplicas = client().admin() + .indices() + .prepareGetSettings("test") + .get() + .getSetting("test", IndexMetadata.SETTING_NUMBER_OF_REPLICAS); + assertEquals("2", numberOfReplicas); + // update setting with null replica will use cluster setting of replica number + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().putNull(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)) + ); + + numberOfReplicas = client().admin() + .indices() + .prepareGetSettings("test") + .get() + .getSetting("test", IndexMetadata.SETTING_NUMBER_OF_REPLICAS); + assertEquals(defaultNumberOfReplica, numberOfReplicas); + + // Clean up cluster setting, then update setting with null replica should pick up default value of 1 + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("cluster.default_number_of_replicas")) + ); + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().putNull(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)) + ); + + numberOfReplicas = client().admin() + .indices() + .prepareGetSettings("test") + .get() + .getSetting("test", IndexMetadata.SETTING_NUMBER_OF_REPLICAS); + assertEquals("1", numberOfReplicas); + } + public void testNoopUpdate() { internalCluster().ensureAtLeastNumDataNodes(2); final ClusterService clusterService = internalCluster().getClusterManagerNodeInstance(ClusterService.class); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index 4e7e31bbb9222..8c350d6b9cef5 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -140,6 +140,7 @@ public void updateSettings( validateRefreshIntervalSettings(normalizedSettings, clusterService.getClusterSettings()); validateTranslogDurabilitySettings(normalizedSettings, clusterService.getClusterSettings(), clusterService.getSettings()); + final int defaultReplicaCount = clusterService.getClusterSettings().get(Metadata.DEFAULT_REPLICA_COUNT_SETTING); Settings.Builder settingsForClosedIndices = Settings.builder(); Settings.Builder settingsForOpenIndices = Settings.builder(); @@ -248,7 +249,10 @@ public ClusterState execute(ClusterState currentState) { } if (IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.exists(openSettings)) { - final int updatedNumberOfReplicas = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(openSettings); + final int updatedNumberOfReplicas = openSettings.getAsInt( + IndexMetadata.SETTING_NUMBER_OF_REPLICAS, + defaultReplicaCount + ); if (preserveExisting == false) { for (Index index : request.indices()) { if (index.getName().charAt(0) != '.') { @@ -329,15 +333,13 @@ public ClusterState execute(ClusterState currentState) { /* * The setting index.number_of_replicas is special; we require that this setting has a value in the index. When * creating the index, we ensure this by explicitly providing a value for the setting to the default (one) if - * there is a not value provided on the source of the index creation. A user can update this setting though, - * including updating it to null, indicating that they want to use the default value. In this case, we again - * have to provide an explicit value for the setting to the default (one). + * there is no value provided on the source of the index creation or "cluster.default_number_of_replicas" is + * not set. A user can update this setting though, including updating it to null, indicating that they want to + * use the value configured by cluster settings or a default value 1. In this case, we again have to provide + * an explicit value for the setting. */ if (IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.exists(indexSettings) == false) { - indexSettings.put( - IndexMetadata.SETTING_NUMBER_OF_REPLICAS, - IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(Settings.EMPTY) - ); + indexSettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, defaultReplicaCount); } Settings finalSettings = indexSettings.build(); indexScopedSettings.validate( From 53c9ddf77c5d050d188fd342035f932600a41608 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Fri, 11 Oct 2024 15:25:34 -0700 Subject: [PATCH 223/357] Remove ApproximateIndexOrDocValuesQuery (#16273) Looking into the query approximation framework, I realized that we don't actually need ApproximateIndexOrDocValuesQuery. We already have ApproximateScoreQuery that is able to choose between the approximate query and the original query. Assuming the range query is over an indexed field, it is (potentially) possible to approximate. If there are doc values, then the original query will be an IndexOrDocValuesQuery. Otherwise, it will just be a PointRangeQuery. Either way, we can wrap the original query into a generic ApproximateScoreQuery along with the approximate query. The approximation logic doesn't need to be specialized to the IndexOrDocValue case. --------- Signed-off-by: Michael Froh --- CHANGELOG.md | 1 + .../index/mapper/DateFieldMapper.java | 59 ++++----- .../bucket/filterrewrite/Helper.java | 4 +- .../ApproximateIndexOrDocValuesQuery.java | 62 ---------- .../ApproximatePointRangeQuery.java | 3 - .../approximate/ApproximateScoreQuery.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 40 ++++--- ...angeFieldQueryStringQueryBuilderTests.java | 18 +-- .../index/mapper/RangeFieldTypeTests.java | 4 +- .../query/MatchPhraseQueryBuilderTests.java | 4 +- .../query/QueryStringQueryBuilderTests.java | 17 +-- .../index/query/RangeQueryBuilderTests.java | 62 +++++----- ...ApproximateIndexOrDocValuesQueryTests.java | 113 ------------------ 13 files changed, 119 insertions(+), 272 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java delete mode 100644 server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a32d2503bbef..43993cd7ca9dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) - Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) +- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 4698440f08036..7fbb38c47572c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -62,8 +62,8 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.search.lookup.SearchLookup; import java.io.IOException; @@ -113,21 +113,6 @@ public static DateFormatter getDefaultDateTimeFormatter() { : LEGACY_DEFAULT_DATE_TIME_FORMATTER; } - public static Query getDefaultQuery(Query pointRangeQuery, Query dvQuery, String name, long l, long u) { - return FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY_SETTING) - ? new ApproximateIndexOrDocValuesQuery( - pointRangeQuery, - new ApproximatePointRangeQuery(name, pack(new long[] { l }).bytes, pack(new long[] { u }).bytes, new long[] { l }.length) { - @Override - protected String toString(int dimension, byte[] value) { - return Long.toString(LongPoint.decodeDimension(value, 0)); - } - }, - dvQuery - ) - : new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); - } - /** * Resolution of the date time * @@ -488,22 +473,42 @@ public Query rangeQuery( } DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - Query pointRangeQuery = isSearchable() ? LongPoint.newRangeQuery(name(), l, u) : null; Query dvQuery = hasDocValues() ? SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u) : null; - if (isSearchable() && hasDocValues()) { - Query query = getDefaultQuery(pointRangeQuery, dvQuery, name(), l, u); - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + if (isSearchable()) { + Query pointRangeQuery = LongPoint.newRangeQuery(name(), l, u); + Query query; + if (dvQuery != null) { + query = new IndexOrDocValuesQuery(pointRangeQuery, dvQuery); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } else { + query = pointRangeQuery; + } + if (FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY_SETTING)) { + return new ApproximateScoreQuery( + query, + new ApproximatePointRangeQuery( + name(), + pack(new long[] { l }).bytes, + pack(new long[] { u }).bytes, + new long[] { l }.length + ) { + @Override + protected String toString(int dimension, byte[] value) { + return Long.toString(LongPoint.decodeDimension(value, 0)); + } + } + ); } return query; } - if (hasDocValues()) { - if (context.indexSortedOnField(name())) { - dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, dvQuery); - } - return dvQuery; + + // Not searchable. Must have doc values. + if (context.indexSortedOnField(name())) { + dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, dvQuery); } - return pointRangeQuery; + return dvQuery; }); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java index 17da7e5712be8..356380a687003 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java @@ -23,7 +23,7 @@ import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.query.DateRangeIncludingNowQuery; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.search.internal.SearchContext; import java.io.IOException; @@ -55,7 +55,7 @@ private Helper() {} queryWrappers.put(FunctionScoreQuery.class, q -> ((FunctionScoreQuery) q).getSubQuery()); queryWrappers.put(DateRangeIncludingNowQuery.class, q -> ((DateRangeIncludingNowQuery) q).getQuery()); queryWrappers.put(IndexOrDocValuesQuery.class, q -> ((IndexOrDocValuesQuery) q).getIndexQuery()); - queryWrappers.put(ApproximateIndexOrDocValuesQuery.class, q -> ((ApproximateIndexOrDocValuesQuery) q).getOriginalQuery()); + queryWrappers.put(ApproximateScoreQuery.class, q -> ((ApproximateScoreQuery) q).getOriginalQuery()); } /** diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java deleted file mode 100644 index b99e0a0cbf808..0000000000000 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.search.approximate; - -import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryVisitor; - -/** - * A wrapper around {@link IndexOrDocValuesQuery} that can be used to run approximate queries. - * It delegates to either {@link ApproximateQuery} or {@link IndexOrDocValuesQuery} based on whether the query can be approximated or not. - * @see ApproximateQuery - */ -public final class ApproximateIndexOrDocValuesQuery extends ApproximateScoreQuery { - - private final ApproximateQuery approximateIndexQuery; - private final IndexOrDocValuesQuery indexOrDocValuesQuery; - - public ApproximateIndexOrDocValuesQuery(Query indexQuery, ApproximateQuery approximateIndexQuery, Query dvQuery) { - super(new IndexOrDocValuesQuery(indexQuery, dvQuery), approximateIndexQuery); - this.approximateIndexQuery = approximateIndexQuery; - this.indexOrDocValuesQuery = new IndexOrDocValuesQuery(indexQuery, dvQuery); - } - - @Override - public String toString(String field) { - return "ApproximateIndexOrDocValuesQuery(indexQuery=" - + indexOrDocValuesQuery.getIndexQuery().toString(field) - + ", approximateIndexQuery=" - + approximateIndexQuery.toString(field) - + ", dvQuery=" - + indexOrDocValuesQuery.getRandomAccessQuery().toString(field) - + ")"; - } - - @Override - public void visit(QueryVisitor visitor) { - indexOrDocValuesQuery.visit(visitor); - } - - @Override - public boolean equals(Object obj) { - if (sameClassAs(obj) == false) { - return false; - } - return true; - } - - @Override - public int hashCode() { - int h = classHash(); - h = 31 * h + indexOrDocValuesQuery.getIndexQuery().hashCode(); - h = 31 * h + indexOrDocValuesQuery.getRandomAccessQuery().hashCode(); - return h; - } -} diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java index 8076da6ab970b..6ff01f5f39d36 100644 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java @@ -433,9 +433,6 @@ public boolean canApproximate(SearchContext context) { if (context.aggregations() != null) { return false; } - if (!(context.query() instanceof ApproximateIndexOrDocValuesQuery)) { - return false; - } // size 0 could be set for caching if (context.from() + context.size() == 0) { this.setSize(10_000); diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java index d1dd32b239f28..2395142c606ae 100644 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java @@ -21,12 +21,12 @@ * Entry-point for the approximation framework. * This class is heavily inspired by {@link org.apache.lucene.search.IndexOrDocValuesQuery}. It acts as a wrapper that consumer two queries, a regular query and an approximate version of the same. By default, it executes the regular query and returns {@link Weight#scorer} for the original query. At run-time, depending on certain constraints, we can re-write the {@code Weight} to use the approximate weight instead. */ -public class ApproximateScoreQuery extends Query { +public final class ApproximateScoreQuery extends Query { private final Query originalQuery; private final ApproximateQuery approximationQuery; - protected Query resolvedQuery; + Query resolvedQuery; public ApproximateScoreQuery(Query originalQuery, ApproximateQuery approximationQuery) { this.originalQuery = originalQuery; diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index 259904fc143a1..15b16f4610062 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -41,6 +41,7 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; @@ -65,8 +66,8 @@ import org.opensearch.index.query.DateRangeIncludingNowQuery; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.joda.time.DateTimeZone; import java.io.IOException; @@ -212,8 +213,11 @@ public void testTermQuery() { MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date)).toInstant().toEpochMilli(); - Query expected = new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant, instant + 999), + Query expected = new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant + 999), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999) + ), new ApproximatePointRangeQuery( "field", pack(new long[] { instant }).bytes, @@ -224,8 +228,7 @@ public void testTermQuery() { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999) + } ); assumeThat( "Using Approximate Range Query as default", @@ -278,8 +281,11 @@ public void testRangeQuery() throws IOException { String date2 = "2016-04-28T11:33:52"; long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli(); long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999; - Query expected = new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant1, instant2), + Query expected = new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ), new ApproximatePointRangeQuery( "field", pack(new long[] { instant1 }).bytes, @@ -290,8 +296,7 @@ public void testRangeQuery() throws IOException { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + } ); assumeThat( "Using Approximate Range Query as default", @@ -306,8 +311,11 @@ protected String toString(int dimension, byte[] value) { instant1 = nowInMillis; instant2 = instant1 + 100; expected = new DateRangeIncludingNowQuery( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant1, instant2), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ), new ApproximatePointRangeQuery( "field", pack(new long[] { instant1 }).bytes, @@ -318,8 +326,7 @@ protected String toString(int dimension, byte[] value) { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + } ) ); assumeThat( @@ -388,8 +395,8 @@ public void testRangeQueryWithIndexSort() { "field", instant1, instant2, - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery("field", instant1, instant2), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery(LongPoint.newRangeQuery("field", instant1, instant2), dvQuery), new ApproximatePointRangeQuery( "field", pack(new long[] { instant1 }).bytes, @@ -400,8 +407,7 @@ public void testRangeQueryWithIndexSort() { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - dvQuery + } ) ); assumeThat( diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java index 7a8ac829bdd97..a11a16f421783 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldQueryStringQueryBuilderTests.java @@ -50,8 +50,8 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryStringQueryBuilder; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -191,11 +191,14 @@ public void testDateRangeQuery() throws Exception { is(true) ); assertEquals( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery( - DATE_FIELD_NAME, - parser.parse(lowerBoundExact, () -> 0).toEpochMilli(), - parser.parse(upperBoundExact, () -> 0).toEpochMilli() + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery( + DATE_FIELD_NAME, + parser.parse(lowerBoundExact, () -> 0).toEpochMilli(), + parser.parse(upperBoundExact, () -> 0).toEpochMilli() + ), + controlDv ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, @@ -207,8 +210,7 @@ public void testDateRangeQuery() throws Exception { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - controlDv + } ), queryOnDateField ); diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java index b157c43e45451..04d4e6b3f852e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java @@ -57,7 +57,7 @@ import org.opensearch.index.mapper.RangeFieldMapper.RangeFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.IndexSettingsModule; import org.joda.time.DateTime; import org.junit.Before; @@ -293,7 +293,7 @@ public void testDateRangeQueryUsingMappingFormatLegacy() { ); assertEquals( "field:[1465975790000 TO 1466062190999]", - ((IndexOrDocValuesQuery) ((ApproximateIndexOrDocValuesQuery) queryOnDateField).getOriginalQuery()).getIndexQuery().toString() + ((IndexOrDocValuesQuery) ((ApproximateScoreQuery) queryOnDateField).getOriginalQuery()).getIndexQuery().toString() ); } diff --git a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java index ddf58073a5206..48f697cff1bea 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchPhraseQueryBuilderTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.search.TermQuery; import org.opensearch.core.common.ParsingException; import org.opensearch.index.search.MatchQuery.ZeroTermsQuery; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -131,7 +131,7 @@ protected void doAssertLuceneQuery(MatchPhraseQueryBuilder queryBuilder, Query q .or(instanceOf(PointRangeQuery.class)) .or(instanceOf(IndexOrDocValuesQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)) - .or(instanceOf(ApproximateIndexOrDocValuesQuery.class)) + .or(instanceOf(ApproximateScoreQuery.class)) ); } diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index 5b030df20e889..6f295100d9e47 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -47,6 +47,7 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -76,8 +77,8 @@ import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.search.QueryStringQueryParser; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; @@ -863,7 +864,7 @@ public void testToQueryDateWithTimeZone() throws Exception { FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), is(true) ); - assertThat(query, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + assertThat(query, instanceOf(ApproximateScoreQuery.class)); long lower = 0; // 1970-01-01T00:00:00.999 UTC long upper = 86399999; // 1970-01-01T23:59:59.999 UTC assertEquals(calculateExpectedDateQuery(lower, upper), query); @@ -872,9 +873,12 @@ public void testToQueryDateWithTimeZone() throws Exception { assertEquals(calculateExpectedDateQuery(lower + msPerHour, upper + msPerHour), qsq.timeZone("-01:00").toQuery(context)); } - private ApproximateIndexOrDocValuesQuery calculateExpectedDateQuery(long lower, long upper) { - return new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + private ApproximateScoreQuery calculateExpectedDateQuery(long lower, long upper) { + return new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, pack(new long[] { lower }).bytes, @@ -885,8 +889,7 @@ private ApproximateIndexOrDocValuesQuery calculateExpectedDateQuery(long lower, protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + } ); } diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index 799d7c7b63462..79f1452db297b 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -55,9 +55,9 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MappedFieldType.Relation; import org.opensearch.index.mapper.MapperService; -import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery; import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.approximate.ApproximateQuery; +import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.test.AbstractQueryTestCase; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; @@ -196,10 +196,10 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), is(true) ); - assertThat(query, instanceOf(ApproximateIndexOrDocValuesQuery.class)); - Query approximationQuery = ((ApproximateIndexOrDocValuesQuery) query).getApproximationQuery(); + assertThat(query, instanceOf(ApproximateScoreQuery.class)); + Query approximationQuery = ((ApproximateScoreQuery) query).getApproximationQuery(); assertThat(approximationQuery, instanceOf(ApproximateQuery.class)); - Query originalQuery = ((ApproximateIndexOrDocValuesQuery) query).getOriginalQuery(); + Query originalQuery = ((ApproximateScoreQuery) query).getOriginalQuery(); assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class)); MapperService mapperService = context.getMapperService(); MappedFieldType mappedFieldType = mapperService.fieldType(expectedFieldName); @@ -250,8 +250,11 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, } } assertEquals( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, minLong, maxLong), + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, minLong, maxLong) + ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, pack(new long[] { minLong }).bytes, @@ -262,8 +265,7 @@ protected void doAssertLuceneQuery(RangeQueryBuilder queryBuilder, Query query, protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, minLong, maxLong) + } ), query ); @@ -336,16 +338,19 @@ public void testDateRangeQueryFormat() throws IOException { FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), is(true) ); - assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); - Query approximationQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getApproximationQuery(); + assertThat(parsedQuery, instanceOf(ApproximateScoreQuery.class)); + Query approximationQuery = ((ApproximateScoreQuery) parsedQuery).getApproximationQuery(); assertThat(approximationQuery, instanceOf(ApproximateQuery.class)); - Query originalQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getOriginalQuery(); + Query originalQuery = ((ApproximateScoreQuery) parsedQuery).getOriginalQuery(); assertThat(originalQuery, instanceOf(IndexOrDocValuesQuery.class)); long lower = DateTime.parse("2012-01-01T00:00:00.000+00").getMillis(); long upper = DateTime.parse("2030-01-01T00:00:00.000+00").getMillis() - 1; assertEquals( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, pack(new long[] { lower }).bytes, @@ -356,8 +361,7 @@ public void testDateRangeQueryFormat() throws IOException { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + } ), parsedQuery ); @@ -394,13 +398,16 @@ public void testDateRangeBoundaries() throws IOException { FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), is(true) ); - assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + assertThat(parsedQuery, instanceOf(ApproximateScoreQuery.class)); long lower = DateTime.parse("2014-11-01T00:00:00.000+00").getMillis(); long upper = DateTime.parse("2014-12-08T23:59:59.999+00").getMillis(); assertEquals( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, pack(new long[] { lower }).bytes, @@ -411,8 +418,7 @@ public void testDateRangeBoundaries() throws IOException { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + } ) , @@ -430,12 +436,15 @@ protected String toString(int dimension, byte[] value) { + " }\n" + "}"; parsedQuery = parseQuery(query).toQuery(createShardContext()); - assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); + assertThat(parsedQuery, instanceOf(ApproximateScoreQuery.class)); lower = DateTime.parse("2014-11-30T23:59:59.999+00").getMillis() + 1; upper = DateTime.parse("2014-12-08T00:00:00.000+00").getMillis() - 1; assertEquals( - new ApproximateIndexOrDocValuesQuery( - LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + new ApproximateScoreQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery(DATE_FIELD_NAME, lower, upper), + SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + ), new ApproximatePointRangeQuery( DATE_FIELD_NAME, pack(new long[] { lower }).bytes, @@ -446,8 +455,7 @@ protected String toString(int dimension, byte[] value) { protected String toString(int dimension, byte[] value) { return Long.toString(LongPoint.decodeDimension(value, 0)); } - }, - SortedNumericDocValuesField.newSlowRangeQuery(DATE_FIELD_NAME, lower, upper) + } ) , @@ -476,8 +484,8 @@ public void testDateRangeQueryTimezone() throws IOException { FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY), is(true) ); - assertThat(parsedQuery, instanceOf(ApproximateIndexOrDocValuesQuery.class)); - parsedQuery = ((ApproximateIndexOrDocValuesQuery) parsedQuery).getApproximationQuery(); + assertThat(parsedQuery, instanceOf(ApproximateScoreQuery.class)); + parsedQuery = ((ApproximateScoreQuery) parsedQuery).getApproximationQuery(); assertThat(parsedQuery, instanceOf(ApproximateQuery.class)); // TODO what else can we assert diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java deleted file mode 100644 index 47f87c6abf629..0000000000000 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQueryTests.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.search.approximate; - -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Weight; -import org.apache.lucene.store.Directory; -import org.opensearch.search.internal.SearchContext; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; - -import static org.apache.lucene.document.LongPoint.pack; - -public class ApproximateIndexOrDocValuesQueryTests extends OpenSearchTestCase { - private Directory dir; - private IndexWriter w; - private DirectoryReader reader; - private IndexSearcher searcher; - - @Before - public void initSearcher() throws IOException { - dir = newDirectory(); - w = new IndexWriter(dir, newIndexWriterConfig()); - } - - @After - public void closeAllTheReaders() throws IOException { - reader.close(); - w.close(); - dir.close(); - } - - public void testApproximateIndexOrDocValuesQueryWeight() throws IOException { - - long l = Long.MIN_VALUE; - long u = Long.MAX_VALUE; - Query indexQuery = LongPoint.newRangeQuery("test-index", l, u); - - ApproximateQuery approximateIndexQuery = new ApproximatePointRangeQuery( - "test-index", - pack(new long[] { l }).bytes, - pack(new long[] { u }).bytes, - new long[] { l }.length - ) { - protected String toString(int dimension, byte[] value) { - return Long.toString(LongPoint.decodeDimension(value, 0)); - } - }; - - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery("test-index", l, u); - - ApproximateIndexOrDocValuesQuery approximateIndexOrDocValuesQuery = new ApproximateIndexOrDocValuesQuery( - indexQuery, - approximateIndexQuery, - dvQuery - ); - - reader = DirectoryReader.open(w); - searcher = newSearcher(reader); - - approximateIndexOrDocValuesQuery.resolvedQuery = indexQuery; - - Weight weight = approximateIndexOrDocValuesQuery.rewrite(searcher).createWeight(searcher, ScoreMode.COMPLETE, 1f); - - assertTrue(weight instanceof ConstantScoreWeight); - - ApproximateQuery approximateIndexQueryCanApproximate = new ApproximatePointRangeQuery( - "test-index", - pack(new long[] { l }).bytes, - pack(new long[] { u }).bytes, - new long[] { l }.length - ) { - protected String toString(int dimension, byte[] value) { - return Long.toString(LongPoint.decodeDimension(value, 0)); - } - - public boolean canApproximate(SearchContext context) { - return true; - } - }; - - ApproximateIndexOrDocValuesQuery approximateIndexOrDocValuesQueryCanApproximate = new ApproximateIndexOrDocValuesQuery( - indexQuery, - approximateIndexQueryCanApproximate, - dvQuery - ); - - approximateIndexOrDocValuesQueryCanApproximate.resolvedQuery = approximateIndexQueryCanApproximate; - - Weight approximateIndexOrDocValuesQueryCanApproximateWeight = approximateIndexOrDocValuesQueryCanApproximate.rewrite(searcher) - .createWeight(searcher, ScoreMode.COMPLETE, 1f); - - // we get ConstantScoreWeight since we're expecting to call ApproximatePointRangeQuery - assertTrue(approximateIndexOrDocValuesQueryCanApproximateWeight instanceof ConstantScoreWeight); - - } -} From 20536eef239031af1161fa341a57597e7219fcd5 Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:35:50 -0700 Subject: [PATCH 224/357] Fix IndicesRequestCacheIt flaky tests (#16276) Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/indices/IndicesRequestCacheIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 108ef14f0fcb4..557f9e19ee424 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -84,6 +84,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; @@ -586,6 +587,7 @@ public void testCacheWithFilteredAlias() throws InterruptedException { .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1)) .build(); String index = "index"; assertAcked( @@ -599,13 +601,14 @@ public void testCacheWithFilteredAlias() throws InterruptedException { ); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); client.prepareIndex(index).setId("1").setRouting("1").setSource("created_at", DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get(); - // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache - ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get(); - OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); refreshAndWaitForReplication(); indexRandomForConcurrentSearch(index); + // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache + ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get(); + OpenSearchAssertions.assertAllSuccessful(forceMergeResponse); + assertCacheState(client, index, 0, 0); SearchResponse r1 = client.prepareSearch(index) @@ -823,7 +826,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo SearchResponse resp = client.prepareSearch(index) .setRequestCache(true) .setQuery(timeoutQueryBuilder) - .setTimeout(TimeValue.ZERO) + .setTimeout(new TimeValue(5, TimeUnit.MILLISECONDS)) .get(); assertTrue(resp.isTimedOut()); RequestCacheStats requestCacheStats = getRequestCacheStats(client, index); From 5d2d3920f53113340df5b35c6bff6bfa9e9e7adb Mon Sep 17 00:00:00 2001 From: Rishab Nahata Date: Mon, 14 Oct 2024 09:39:49 +0530 Subject: [PATCH 225/357] Skip unnecessary string format in RemoteStoreMigrationAllocationDecider when not in debug mode (#16299) Signed-off-by: Rishab Nahata --- ...RemoteStoreMigrationAllocationDecider.java | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java index 67fe4ea1dcb1b..3da6288292181 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java @@ -89,7 +89,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision( Decision.YES, NAME, - getDecisionDetails(true, shardRouting, targetNode, " for strict compatibility mode") + getDecisionDetails(true, shardRouting, targetNode, " for strict compatibility mode", allocation) ); } @@ -103,11 +103,15 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision( isNoDecision ? Decision.NO : Decision.YES, NAME, - getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason) + getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason, allocation) ); } else if (migrationDirection.equals(Direction.DOCREP)) { // docrep migration direction is currently not supported - return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction")); + return allocation.decision( + Decision.YES, + NAME, + getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction", allocation) + ); } else { // check for remote store backed indices if (remoteSettingsBackedIndex && targetNode.isRemoteStoreNode() == false) { @@ -115,7 +119,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing String reason = new StringBuilder(" because a remote store backed index's shard copy can only be ").append( (shardRouting.assignedToNode() == false) ? "allocated" : "relocated" ).append(" to a remote node").toString(); - return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason)); + return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason, allocation)); } if (shardRouting.primary()) { @@ -128,9 +132,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing // handle scenarios for allocation of a new shard's primary copy private Decision primaryShardDecision(ShardRouting primaryShardRouting, DiscoveryNode targetNode, RoutingAllocation allocation) { if (targetNode.isRemoteStoreNode() == false) { - return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, primaryShardRouting, targetNode, "")); + return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, primaryShardRouting, targetNode, "", allocation)); } - return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, primaryShardRouting, targetNode, "")); + return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, primaryShardRouting, targetNode, "", allocation)); } // Checks if primary shard is on a remote node. @@ -150,20 +154,41 @@ private Decision replicaShardDecision(ShardRouting replicaShardRouting, Discover return allocation.decision( Decision.NO, NAME, - getDecisionDetails(false, replicaShardRouting, targetNode, " since primary shard copy is not yet migrated to remote") + getDecisionDetails( + false, + replicaShardRouting, + targetNode, + " since primary shard copy is not yet migrated to remote", + allocation + ) ); } return allocation.decision( Decision.YES, NAME, - getDecisionDetails(true, replicaShardRouting, targetNode, " since primary shard copy has been migrated to remote") + getDecisionDetails( + true, + replicaShardRouting, + targetNode, + " since primary shard copy has been migrated to remote", + allocation + ) ); } - return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, replicaShardRouting, targetNode, "")); + return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, replicaShardRouting, targetNode, "", allocation)); } // get detailed reason for the decision - private String getDecisionDetails(boolean isYes, ShardRouting shardRouting, DiscoveryNode targetNode, String reason) { + private String getDecisionDetails( + boolean isYes, + ShardRouting shardRouting, + DiscoveryNode targetNode, + String reason, + RoutingAllocation allocation + ) { + if (allocation.debugDecision() == false) { + return null; + } return new StringBuilder("[").append(migrationDirection.direction) .append(" migration_direction]: ") .append(shardRouting.primary() ? "primary" : "replica") From 2166b449d99e5afd9a8d943f7245de2f2635fe5c Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Mon, 14 Oct 2024 10:57:48 +0530 Subject: [PATCH 226/357] Optimise clone operation for incremental full cluster snapshots (#16296) * Optimise clone operation for incremental full cluster snapshots Signed-off-by: Ashish Singh * Add UTs Signed-off-by: Ashish Singh * Add CHANGELOG Signed-off-by: Ashish Singh --------- Signed-off-by: Ashish Singh --- CHANGELOG.md | 2 +- .../snapshots/SnapshotsService.java | 121 ++--- .../snapshots/SnapshotsServiceTests.java | 439 ++++++++++++++++++ 3 files changed, 504 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43993cd7ca9dd..929b3561cdbb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) - Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) - Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) - +- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 541a87c200883..2df094ebe540b 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -1327,7 +1327,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS private final Set currentlyCloning = Collections.synchronizedSet(new HashSet<>()); - private void runReadyClone( + // Made to package private to be able to test the method in UTs + void runReadyClone( Snapshot target, SnapshotId sourceSnapshot, ShardSnapshotStatus shardStatusBefore, @@ -1351,69 +1352,75 @@ public void onFailure(Exception e) { @Override protected void doRun() { final String localNodeId = clusterService.localNode().getId(); - repository.getRepositoryData(ActionListener.wrap(repositoryData -> { - try { - final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData( - repositoryData, + if (remoteStoreIndexShallowCopy == false) { + executeClone(localNodeId, false); + } else { + repository.getRepositoryData(ActionListener.wrap(repositoryData -> { + try { + final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData( + repositoryData, + sourceSnapshot, + repoShardId.index() + ); + final boolean cloneRemoteStoreIndexShardSnapshot = indexMetadata.getSettings() + .getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); + executeClone(localNodeId, cloneRemoteStoreIndexShardSnapshot); + } catch (IOException e) { + logger.warn("Failed to get index-metadata from repository data for index [{}]", repoShardId.index().getName()); + failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); + } + }, this::onFailure)); + } + } + + private void executeClone(String localNodeId, boolean cloneRemoteStoreIndexShardSnapshot) { + if (currentlyCloning.add(repoShardId)) { + if (cloneRemoteStoreIndexShardSnapshot) { + repository.cloneRemoteStoreIndexShardSnapshot( sourceSnapshot, - repoShardId.index() + target.getSnapshotId(), + repoShardId, + shardStatusBefore.generation(), + remoteStoreLockManagerFactory, + getCloneCompletionListener(localNodeId) ); - final boolean cloneRemoteStoreIndexShardSnapshot = remoteStoreIndexShallowCopy - && indexMetadata.getSettings().getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); - final SnapshotId targetSnapshot = target.getSnapshotId(); - final ActionListener listener = ActionListener.wrap( - generation -> innerUpdateSnapshotState( - new ShardSnapshotUpdate( - target, - repoShardId, - new ShardSnapshotStatus(localNodeId, ShardState.SUCCESS, generation) - ), - ActionListener.runBefore( - ActionListener.wrap( - v -> logger.trace( - "Marked [{}] as successfully cloned from [{}] to [{}]", - repoShardId, - sourceSnapshot, - targetSnapshot - ), - e -> { - logger.warn("Cluster state update after successful shard clone [{}] failed", repoShardId); - failAllListenersOnMasterFailOver(e); - } - ), - () -> currentlyCloning.remove(repoShardId) - ) - ), - e -> { - logger.warn("Exception [{}] while trying to clone shard [{}]", e, repoShardId); - failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); - } + } else { + repository.cloneShardSnapshot( + sourceSnapshot, + target.getSnapshotId(), + repoShardId, + shardStatusBefore.generation(), + getCloneCompletionListener(localNodeId) ); - if (currentlyCloning.add(repoShardId)) { - if (cloneRemoteStoreIndexShardSnapshot) { - repository.cloneRemoteStoreIndexShardSnapshot( - sourceSnapshot, - targetSnapshot, + } + } + } + + private ActionListener getCloneCompletionListener(String localNodeId) { + return ActionListener.wrap( + generation -> innerUpdateSnapshotState( + new ShardSnapshotUpdate(target, repoShardId, new ShardSnapshotStatus(localNodeId, ShardState.SUCCESS, generation)), + ActionListener.runBefore( + ActionListener.wrap( + v -> logger.trace( + "Marked [{}] as successfully cloned from [{}] to [{}]", repoShardId, - shardStatusBefore.generation(), - remoteStoreLockManagerFactory, - listener - ); - } else { - repository.cloneShardSnapshot( sourceSnapshot, - targetSnapshot, - repoShardId, - shardStatusBefore.generation(), - listener - ); - } - } - } catch (IOException e) { - logger.warn("Failed to get index-metadata from repository data for index [{}]", repoShardId.index().getName()); + target.getSnapshotId() + ), + e -> { + logger.warn("Cluster state update after successful shard clone [{}] failed", repoShardId); + failAllListenersOnMasterFailOver(e); + } + ), + () -> currentlyCloning.remove(repoShardId) + ) + ), + e -> { + logger.warn("Exception [{}] while trying to clone shard [{}]", e, repoShardId); failCloneShardAndUpdateClusterState(target, sourceSnapshot, repoShardId); } - }, this::onFailure)); + ); } }); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java index 1652ba29b8fa7..e374636f60d22 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java @@ -33,24 +33,41 @@ package org.opensearch.snapshots; import org.opensearch.Version; +import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.repositories.IndexId; +import org.opensearch.repositories.IndexMetaDataGenerations; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.RepositoryShardId; +import org.opensearch.repositories.ShardGenerations; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -58,8 +75,17 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import org.mockito.ArgumentCaptor; + import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class SnapshotsServiceTests extends OpenSearchTestCase { @@ -404,6 +430,419 @@ public void testCompletedCloneStartsNextClone() throws Exception { assertIsNoop(updatedClusterState, completeShardClone); } + /** + * Tests the runReadyClone method when remote store index shallow copy is disabled. + * It verifies that: + * 1. getRepositoryData is never called + * 2. cloneShardSnapshot is called with the correct parameters + */ + public void testRunReadyCloneWithRemoteStoreIndexShallowCopyDisabled() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + try (SnapshotsService snapshotsService = createSnapshotsService()) { + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, false); + verify(mockRepository, never()).getRepositoryData(any()); + verify(mockRepository).cloneShardSnapshot( + eq(sourceSnapshot), + eq(target.getSnapshotId()), + eq(repoShardId), + eq(shardStatusBefore.generation()), + any() + ); + } + } + + /** + * Tests the runReadyClone method when remote store index shallow copy is enabled. + * It verifies that: + * 1. getRepositoryData is called + * 2. cloneRemoteStoreIndexShardSnapshot is called with the correct parameters + * This test simulates a scenario where the index has remote store enabled. + */ + + public void testRunReadyCloneWithRemoteStoreIndexShallowCopyEnabled() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + // Create a real RepositoryData instance + RepositoryData repositoryData = new RepositoryData( + RepositoryData.EMPTY_REPO_GEN, + Collections.singletonMap(sourceSnapshot.getName(), sourceSnapshot), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY + ); + + // Mock the getRepositoryData method to use the ActionListener + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(repositoryData); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + IndexMetadata mockIndexMetadata = mock(IndexMetadata.class); + when(mockRepository.getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any())).thenReturn(mockIndexMetadata); + + Settings mockSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(); + when(mockIndexMetadata.getSettings()).thenReturn(mockSettings); + + try (SnapshotsService snapshotsService = createSnapshotsService()) { + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + // Verify that getRepositoryData was called + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + + // Verify that cloneRemoteStoreIndexShardSnapshot was called with the correct arguments + verify(mockRepository).cloneRemoteStoreIndexShardSnapshot( + eq(sourceSnapshot), + eq(target.getSnapshotId()), + eq(repoShardId), + eq(shardStatusBefore.generation()), + any(), + any() + ); + } + } + + /** + * Tests the error handling in runReadyClone when a RepositoryException occurs. + * It verifies that: + * 1. getRepositoryData is called and throws an exception + * 2. Neither cloneShardSnapshot nor cloneRemoteStoreIndexShardSnapshot are called + */ + public void testRunReadyCloneWithRepositoryException() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + // Mock the getRepositoryData method to throw an exception + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onFailure(new RepositoryException(repoName, "Test exception")); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + try (SnapshotsService snapshotsService = createSnapshotsService()) { + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + // Verify that getRepositoryData was called + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + + // Verify that neither cloneShardSnapshot nor cloneRemoteStoreIndexShardSnapshot were called + verify(mockRepository, never()).cloneShardSnapshot(any(), any(), any(), any(), any()); + verify(mockRepository, never()).cloneRemoteStoreIndexShardSnapshot(any(), any(), any(), any(), any(), any()); + } + } + + /** + * Tests the runReadyClone method when remote store index shallow copy is globally enabled, + * but disabled for the specific index. + * It verifies that: + * 1. getRepositoryData is called + * 2. cloneShardSnapshot is called instead of cloneRemoteStoreIndexShardSnapshot + */ + public void testRunReadyCloneWithRemoteStoreIndexShallowCopyEnabledButIndexDisabled() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + RepositoryData repositoryData = new RepositoryData( + RepositoryData.EMPTY_REPO_GEN, + Collections.singletonMap(sourceSnapshot.getName(), sourceSnapshot), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY + ); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(repositoryData); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + IndexMetadata mockIndexMetadata = mock(IndexMetadata.class); + when(mockRepository.getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any())).thenReturn(mockIndexMetadata); + + Settings mockSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false).build(); + when(mockIndexMetadata.getSettings()).thenReturn(mockSettings); + + try (SnapshotsService snapshotsService = createSnapshotsService()) { + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + verify(mockRepository).cloneShardSnapshot( + eq(sourceSnapshot), + eq(target.getSnapshotId()), + eq(repoShardId), + eq(shardStatusBefore.generation()), + any() + ); + } + } + + /** + * Tests the error handling in runReadyClone when an IOException occurs while getting snapshot index metadata. + * It verifies that: + * 1. getRepositoryData is called + * 2. getSnapshotIndexMetaData throws an IOException + * 3. Neither cloneShardSnapshot nor cloneRemoteStoreIndexShardSnapshot are called + */ + public void testRunReadyCloneWithIOException() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + RepositoryData repositoryData = new RepositoryData( + RepositoryData.EMPTY_REPO_GEN, + Collections.singletonMap(sourceSnapshot.getName(), sourceSnapshot), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY + ); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(repositoryData); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + when(mockRepository.getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any())).thenThrow( + new IOException("Test IO Exception") + ); + + try (SnapshotsService snapshotsService = createSnapshotsService()) { + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + verify(mockRepository).getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any()); + verify(mockRepository, never()).cloneShardSnapshot(any(), any(), any(), any(), any()); + verify(mockRepository, never()).cloneRemoteStoreIndexShardSnapshot(any(), any(), any(), any(), any(), any()); + } + } + + /** + * Tests the completion listener functionality in runReadyClone when the operation is successful. + * It verifies that: + * 1. The clone operation completes successfully + * 2. The correct ShardSnapshotUpdate is submitted to the cluster state + */ + public void testRunReadyCloneCompletionListener() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + RepositoryData repositoryData = new RepositoryData( + RepositoryData.EMPTY_REPO_GEN, + Collections.singletonMap(sourceSnapshot.getName(), sourceSnapshot), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY + ); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(repositoryData); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + IndexMetadata mockIndexMetadata = mock(IndexMetadata.class); + when(mockRepository.getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any())).thenReturn(mockIndexMetadata); + + Settings mockSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(); + when(mockIndexMetadata.getSettings()).thenReturn(mockSettings); + + String newGeneration = "new_generation"; + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(5); + listener.onResponse(newGeneration); + return null; + }).when(mockRepository).cloneRemoteStoreIndexShardSnapshot(any(), any(), any(), any(), any(), any(ActionListener.class)); + + ClusterService mockClusterService = mock(ClusterService.class); + DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + when(mockClusterService.localNode()).thenReturn(localNode); + + SnapshotsService snapshotsService = createSnapshotsService(mockClusterService); + + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + verify(mockRepository).cloneRemoteStoreIndexShardSnapshot( + eq(sourceSnapshot), + eq(target.getSnapshotId()), + eq(repoShardId), + eq(shardStatusBefore.generation()), + any(), + any() + ); + + // Verify that innerUpdateSnapshotState was called with the correct arguments + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass( + SnapshotsService.ShardSnapshotUpdate.class + ); + verify(mockClusterService).submitStateUpdateTask(eq("update snapshot state"), updateCaptor.capture(), any(), any(), any()); + + SnapshotsService.ShardSnapshotUpdate capturedUpdate = updateCaptor.getValue(); + SnapshotsInProgress.ShardSnapshotStatus expectedStatus = new SnapshotsInProgress.ShardSnapshotStatus( + localNode.getId(), + SnapshotsInProgress.ShardState.SUCCESS, + newGeneration + ); + SnapshotsService.ShardSnapshotUpdate expectedUpdate = new SnapshotsService.ShardSnapshotUpdate(target, repoShardId, expectedStatus); + assertEquals(expectedUpdate.hashCode(), capturedUpdate.hashCode()); + } + + /** + * Tests the completion listener functionality in runReadyClone when the operation fails. + * It verifies that: + * 1. The clone operation fails with an exception + * 2. The correct failed ShardSnapshotUpdate is submitted to the cluster state + */ + public void testRunReadyCloneCompletionListenerFailure() throws Exception { + String repoName = "test-repo"; + Snapshot target = snapshot(repoName, "target-snapshot"); + SnapshotId sourceSnapshot = new SnapshotId("source-snapshot", uuid()); + RepositoryShardId repoShardId = new RepositoryShardId(indexId("test-index"), 0); + SnapshotsInProgress.ShardSnapshotStatus shardStatusBefore = initShardStatus(uuid()); + + Repository mockRepository = mock(Repository.class); + + RepositoryData repositoryData = new RepositoryData( + RepositoryData.EMPTY_REPO_GEN, + Collections.singletonMap(sourceSnapshot.getName(), sourceSnapshot), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY + ); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(0); + listener.onResponse(repositoryData); + return null; + }).when(mockRepository).getRepositoryData(any(ActionListener.class)); + + IndexMetadata mockIndexMetadata = mock(IndexMetadata.class); + when(mockRepository.getSnapshotIndexMetaData(eq(repositoryData), eq(sourceSnapshot), any())).thenReturn(mockIndexMetadata); + + Settings mockSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true).build(); + when(mockIndexMetadata.getSettings()).thenReturn(mockSettings); + + Exception testException = new RuntimeException("Test exception"); + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(5); + listener.onFailure(testException); + return null; + }).when(mockRepository).cloneRemoteStoreIndexShardSnapshot(any(), any(), any(), any(), any(), any(ActionListener.class)); + + ClusterService mockClusterService = mock(ClusterService.class); + DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + when(mockClusterService.localNode()).thenReturn(localNode); + + SnapshotsService snapshotsService = createSnapshotsService(mockClusterService); + + snapshotsService.runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, mockRepository, true); + + verify(mockRepository).getRepositoryData(any(ActionListener.class)); + verify(mockRepository).cloneRemoteStoreIndexShardSnapshot( + eq(sourceSnapshot), + eq(target.getSnapshotId()), + eq(repoShardId), + eq(shardStatusBefore.generation()), + any(), + any() + ); + + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass( + SnapshotsService.ShardSnapshotUpdate.class + ); + verify(mockClusterService).submitStateUpdateTask(eq("update snapshot state"), updateCaptor.capture(), any(), any(), any()); + + SnapshotsService.ShardSnapshotUpdate capturedUpdate = updateCaptor.getValue(); + SnapshotsInProgress.ShardSnapshotStatus expectedStatus = new SnapshotsInProgress.ShardSnapshotStatus( + localNode.getId(), + SnapshotsInProgress.ShardState.FAILED, + "failed to clone shard snapshot", + null + ); + SnapshotsService.ShardSnapshotUpdate expectedUpdate = new SnapshotsService.ShardSnapshotUpdate(target, repoShardId, expectedStatus); + assertEquals(expectedUpdate.hashCode(), capturedUpdate.hashCode()); + } + + /** + * Helper method to create a SnapshotsService instance with a provided ClusterService. + * This method mocks all necessary dependencies for the SnapshotsService. + */ + private SnapshotsService createSnapshotsService(ClusterService mockClusterService) { + ThreadPool mockThreadPool = mock(ThreadPool.class); + when(mockThreadPool.executor(ThreadPool.Names.SNAPSHOT)).thenReturn(OpenSearchExecutors.newDirectExecutorService()); + + RepositoriesService mockRepoService = mock(RepositoriesService.class); + TransportService mockTransportService = mock(TransportService.class); + when(mockTransportService.getThreadPool()).thenReturn(mockThreadPool); + + DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + when(mockClusterService.localNode()).thenReturn(localNode); + + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(mockClusterService.getClusterSettings()).thenReturn(clusterSettings); + + return new SnapshotsService( + Settings.EMPTY, + mockClusterService, + mock(IndexNameExpressionResolver.class), + mockRepoService, + mockTransportService, + mock(ActionFilters.class), + null, + mock(RemoteStoreSettings.class) + ); + } + + /** + * Helper method to create a SnapshotsService instance with a mocked ClusterService. + * This method is a convenience wrapper around createSnapshotsService(ClusterService). + */ + private SnapshotsService createSnapshotsService() { + ClusterService mockClusterService = mock(ClusterService.class); + return createSnapshotsService(mockClusterService); + } + private static DiscoveryNodes discoveryNodes(String localNodeId) { return DiscoveryNodes.builder().localNodeId(localNodeId).build(); } From 55e98ed2735d898d884295c00f869f5f3278c222 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 14 Oct 2024 21:00:49 +0800 Subject: [PATCH 227/357] Fix multi-search with template doesn't return status code (#16265) * Fix multi-search with template doesn't return status code Signed-off-by: Gao Binlong * Modify changelog Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../mustache/MultiSearchTemplateResponse.java | 3 + .../mustache/SearchTemplateResponse.java | 5 +- .../MultiSearchTemplateResponseTests.java | 79 +++++++++++++++++++ .../mustache/SearchTemplateResponseTests.java | 1 + .../50_multi_search_template.yml | 48 +++++++++++ 6 files changed, 136 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929b3561cdbb0..bc6c8b024c692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) +- Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) ### Security diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java index ead93158b421c..a84dd85de98ff 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java @@ -32,6 +32,7 @@ package org.opensearch.script.mustache; +import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; import org.opensearch.action.search.MultiSearchResponse; import org.opensearch.common.Nullable; @@ -167,6 +168,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par if (item.isFailure()) { builder.startObject(); OpenSearchException.generateFailureXContent(builder, params, item.getFailure(), true); + builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus()); builder.endObject(); } else { item.getResponse().toXContent(builder, params); @@ -179,6 +181,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par static final class Fields { static final String RESPONSES = "responses"; + static final String STATUS = "status"; } public static MultiSearchTemplateResponse fromXContext(XContentParser parser) { diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java index 9cb6ac127786a..534b70cf80f73 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java @@ -120,7 +120,10 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (hasResponse()) { - response.toXContent(builder, params); + builder.startObject(); + response.innerToXContent(builder, params); + builder.field(MultiSearchTemplateResponse.Fields.STATUS, response.status().getStatus()); + builder.endObject(); } else { builder.startObject(); // we can assume the template is always json as we convert it before compiling it diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java index aeb7cdc2c6a28..ffd282e4fedc0 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java @@ -34,8 +34,12 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.test.AbstractXContentTestCase; @@ -44,6 +48,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -177,4 +182,78 @@ public void testFromXContentWithFailures() throws IOException { ToXContent.EMPTY_PARAMS ); } + + public void testToXContentWithFailures() throws Exception { + long overallTookInMillis = randomNonNegativeLong(); + MultiSearchTemplateResponse.Item[] items = new MultiSearchTemplateResponse.Item[2]; + + long tookInMillis = 1L; + int totalShards = 2; + int successfulShards = 2; + int skippedShards = 0; + InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); + SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse(); + SearchResponse searchResponse = new SearchResponse( + internalSearchResponse, + null, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + searchTemplateResponse.setResponse(searchResponse); + items[0] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null); + + items[1] = new MultiSearchTemplateResponse.Item(null, new IllegalArgumentException("Invalid argument")); + + MultiSearchTemplateResponse response = new MultiSearchTemplateResponse(items, overallTookInMillis); + + XContentType contentType = randomFrom(XContentType.values()); + XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType) + .startObject() + .field("took", overallTookInMillis) + .startArray("responses") + .startObject() + .field("took", 1) + .field("timed_out", false) + .startObject("_shards") + .field("total", 2) + .field("successful", 2) + .field("skipped", 0) + .field("failed", 0) + .endObject() + .startObject("hits") + .startObject("total") + .field("value", 0) + .field("relation", "eq") + .endObject() + .field("max_score", 0.0F) + .startArray("hits") + .endArray() + .endObject() + .field("status", 200) + .endObject() + .startObject() + .startObject("error") + .field("type", "illegal_argument_exception") + .field("reason", "Invalid argument") + .startArray("root_cause") + .startObject() + .field("type", "illegal_argument_exception") + .field("reason", "Invalid argument") + .endObject() + .endArray() + .endObject() + .field("status", 400) + .endObject() + .endArray() + .endObject(); + + XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); + response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); + + assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType); + } } diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java index c2685e45ecb6b..c00751d8ef758 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java @@ -234,6 +234,7 @@ public void testSearchResponseToXContent() throws IOException { .endObject() .endArray() .endObject() + .field("status", 200) .endObject(); XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); diff --git a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index e92e10b9ad276..2b99fa633e640 100644 --- a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -205,3 +205,51 @@ setup: - match: { responses.1.hits.total.relation: eq } - match: { responses.2.hits.total.value: 1 } - match: { responses.1.hits.total.relation: eq } + +--- +"Test multi-search template returns status code": + - skip: + version: " - 2.17.99" + reason: Fixed in 2.18.0. + - do: + msearch_template: + rest_total_hits_as_int: true + body: + # Search 0 is OK + - index: index_* + - source: '{"query": {"match": {"foo": "{{value}}"} } }' + params: + value: "foo" + # Search 1 has an unclosed JSON template + - index: index_* + - source: '{"query": {"match": {' + params: + field: "foo" + value: "bar" + # Search 2 is OK + - index: _all + - source: + query: + match: {foo: "{{text}}"} + params: + text: "baz" + # Search 3 has an unknown query type + - index: index_* + - source: '{"query": {"{{query_type}}": {} }' # Unknown query type + params: + query_type: "unknown" + # Search 4 has an unsupported track_total_hits + - index: index_* + - source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }' + params: + trackTotalHits: 1 + # Search 5 has an unknown index + - index: unknown_index + - source: '{"query": {"match_all": {} } }' + + - match: { responses.0.status: 200 } + - match: { responses.1.status: 400 } + - match: { responses.2.status: 200 } + - match: { responses.3.status: 400 } + - match: { responses.4.status: 400 } + - match: { responses.5.status: 404 } From b5dcde37b5b6122d7d8533a7aa10920d62c42ea0 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 14 Oct 2024 18:40:27 +0530 Subject: [PATCH 228/357] [Snapshot V2] Move timestamp pinning before cluster state update (#16269) * Move timestamp pinning before cluster state update Signed-off-by: Gaurav Bafna * Address PR Comments Signed-off-by: Gaurav Bafna * Fix IT Signed-off-by: Gaurav Bafna --------- Signed-off-by: Gaurav Bafna --- .../remotestore/RemoteRestoreSnapshotIT.java | 27 ++-- .../snapshots/SnapshotsService.java | 149 ++++++++---------- 2 files changed, 80 insertions(+), 96 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 927dbf9995778..70e283791fc3e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -938,17 +938,8 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio Thread thread = new Thread(() -> { try { String snapshotName = "snapshot-concurrent-" + snapshotIndex; - CreateSnapshotResponse createSnapshotResponse2 = client().admin() - .cluster() - .prepareCreateSnapshot(snapshotRepoName, snapshotName) - .setWaitForCompletion(true) - .get(); - SnapshotInfo snapshotInfo = createSnapshotResponse2.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); - assertThat(snapshotInfo.successfulShards(), greaterThan(0)); - assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); - assertThat(snapshotInfo.snapshotId().getName(), equalTo(snapshotName)); - assertThat(snapshotInfo.getPinnedTimestamp(), greaterThan(0L)); + client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, snapshotName).setWaitForCompletion(true).get(); + logger.info("Snapshot completed {}", snapshotName); } catch (Exception e) {} }); threads.add(thread); @@ -963,15 +954,19 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio thread.join(); } - // Validate that only one snapshot has been created + // Sleeping 10 sec for earlier created snapshot to complete runNextQueuedOperation and be ready for next snapshot + // We can't put `waitFor` since we don't have visibility on its completion + Thread.sleep(TimeValue.timeValueSeconds(10).seconds()); + client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, "snapshot-cleanup-timestamp").setWaitForCompletion(true).get(); Repository repository = internalCluster().getInstance(RepositoriesService.class).repository(snapshotRepoName); PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); repository.getRepositoryData(repositoryDataPlainActionFuture); - RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); - assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1)); - forceSyncPinnedTimestamps(); - assertEquals(RemoteStorePinnedTimestampService.getPinnedEntities().size(), repositoryData.getSnapshotIds().size()); + assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(2)); + waitUntil(() -> { + forceSyncPinnedTimestamps(); + return RemoteStorePinnedTimestampService.getPinnedEntities().size() == repositoryData.getSnapshotIds().size(); + }); } public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception { diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 2df094ebe540b..6688c7dd0431a 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -79,6 +79,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.Priority; +import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; @@ -466,34 +467,35 @@ public TimeValue timeout() { * @param listener snapshot creation listener */ public void createSnapshotV2(final CreateSnapshotRequest request, final ActionListener listener) { - long pinnedTimestamp = System.currentTimeMillis(); final String repositoryName = request.repository(); final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot()); + validate(repositoryName, snapshotName); + + final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot + Snapshot snapshot = new Snapshot(repositoryName, snapshotId); + long pinnedTimestamp = System.currentTimeMillis(); + try { + updateSnapshotPinnedTimestamp(snapshot, pinnedTimestamp); + } catch (Exception e) { + listener.onFailure(e); + return; + } Repository repository = repositoriesService.repository(repositoryName); - validate(repositoryName, snapshotName); repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) { private SnapshotsInProgress.Entry newEntry; - - private SnapshotId snapshotId; - - private Snapshot snapshot; - boolean enteredLoop; @Override public ClusterState execute(ClusterState currentState) { // move to in progress - snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot Repository repository = repositoriesService.repository(repositoryName); - if (repository.isReadOnly()) { listener.onFailure( new RepositoryException(repository.getMetadata().name(), "cannot create snapshot-v2 in a readonly repository") ); } - snapshot = new Snapshot(repositoryName, snapshotId); final Map userMeta = repository.adaptUserMetadata(request.userMetadata()); createSnapshotPreValidations(currentState, repositoryData, repositoryName, snapshotName); @@ -593,59 +595,46 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl pinnedTimestamp ); final Version version = minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null); - final StepListener pinnedTimestampListener = new StepListener<>(); - pinnedTimestampListener.whenComplete(repoData -> { - repository.finalizeSnapshot( - shardGenerations, - repositoryData.getGenId(), - metadataForSnapshot(newState.metadata(), request.includeGlobalState(), false, dataStreams, newEntry.indices()), - snapshotInfo, - version, - state -> stateWithoutSnapshot(state, snapshot), - Priority.IMMEDIATE, - new ActionListener() { - @Override - public void onResponse(RepositoryData repositoryData) { - if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { - leaveRepoLoop(repositoryName); - failSnapshotCompletionListeners( - snapshot, - new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") - ); - listener.onFailure( - new SnapshotException( - repositoryName, - snapshotName, - "Aborting snapshot-v2, no longer cluster manager" - ) - ); - return; - } - listener.onResponse(snapshotInfo); - // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations - // can get queued . This is triggering them. - runNextQueuedOperation(repositoryData, repositoryName, true); - cleanOrphanTimestamp(repositoryName, repositoryData); - } - - @Override - public void onFailure(Exception e) { - logger.error("Failed to finalize snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); + repository.finalizeSnapshot( + shardGenerations, + repositoryData.getGenId(), + metadataForSnapshot(newState.metadata(), request.includeGlobalState(), false, dataStreams, newEntry.indices()), + snapshotInfo, + version, + state -> stateWithoutSnapshot(state, snapshot), + Priority.IMMEDIATE, + new ActionListener() { + @Override + public void onResponse(RepositoryData repositoryData) { + if (clusterService.state().nodes().isLocalNodeElectedClusterManager() == false) { leaveRepoLoop(repositoryName); - // cleaning up in progress snapshot here - stateWithoutSnapshotV2(newState); - listener.onFailure(e); + failSnapshotCompletionListeners( + snapshot, + new SnapshotException(snapshot, "Aborting snapshot-v2, no longer cluster manager") + ); + listener.onFailure( + new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2, no longer cluster manager") + ); + return; } + listener.onResponse(snapshotInfo); + logger.info("created snapshot-v2 [{}] in repository [{}]", repositoryName, snapshotName); + // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations + // can get queued . This is triggering them. + runNextQueuedOperation(repositoryData, repositoryName, true); + cleanOrphanTimestamp(repositoryName, repositoryData); } - ); - }, e -> { - logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} {} ", repositoryName, snapshotName, e); - leaveRepoLoop(repositoryName); - // cleaning up in progress snapshot here - stateWithoutSnapshotV2(newState); - listener.onFailure(e); - }); - updateSnapshotPinnedTimestamp(repositoryData, snapshot, pinnedTimestamp, pinnedTimestampListener); + + @Override + public void onFailure(Exception e) { + logger.error("Failed to finalize snapshot repo {} for snapshot-v2 {} ", repositoryName, snapshotName); + leaveRepoLoop(repositoryName); + // cleaning up in progress snapshot here + stateWithoutSnapshotV2(newState); + listener.onFailure(e); + } + } + ); } @Override @@ -733,30 +722,30 @@ private void createSnapshotPreValidations( ensureNoCleanupInProgress(currentState, repositoryName, snapshotName); } - private void updateSnapshotPinnedTimestamp( - RepositoryData repositoryData, - Snapshot snapshot, - long timestampToPin, - ActionListener listener - ) { + private void updateSnapshotPinnedTimestamp(Snapshot snapshot, long timestampToPin) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + SetOnce ex = new SetOnce<>(); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(Void unused) { + logger.debug("Timestamp pinned successfully for snapshot {}", snapshot.getSnapshotId().getName()); + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to pin timestamp for snapshot {} with exception {}", snapshot.getSnapshotId().getName(), e); + ex.set(e); + } + }; remoteStorePinnedTimestampService.pinTimestamp( timestampToPin, getPinningEntity(snapshot.getRepository(), snapshot.getSnapshotId().getUUID()), - new ActionListener() { - @Override - public void onResponse(Void unused) { - logger.debug("Timestamp pinned successfully for snapshot {}", snapshot.getSnapshotId().getName()); - listener.onResponse(repositoryData); - } - - @Override - public void onFailure(Exception e) { - logger.error("Failed to pin timestamp for snapshot {} with exception {}", snapshot.getSnapshotId().getName(), e); - listener.onFailure(e); - - } - } + new LatchedActionListener<>(listener, latch) ); + latch.await(); + if (ex.get() != null) { + throw ex.get(); + } } public static String getPinningEntity(String repositoryName, String snapshotUUID) { From 7065e6b0bc89028a742e89f30361c60c7c8a9da2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:28:24 -0400 Subject: [PATCH 229/357] Bump ch.qos.logback:logback-core from 1.5.8 to 1.5.10 in /test/fixtures/hdfs-fixture (#16307) * Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.8 to 1.5.10. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.8...v_1.5.10) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6c8b024c692..84aae800bb50a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) -- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) - Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b3a90b5de2589..65fb55afbc7bd 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.8" + api "ch.qos.logback:logback-core:1.5.10" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.27.0' From f4bf0dae6ad08655bf68da73deeb3eb55563608c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:33:38 -0400 Subject: [PATCH 230/357] Bump me.champeau.gradle.japicmp from 0.4.3 to 0.4.4 in /server (#16309) * Bump me.champeau.gradle.japicmp from 0.4.3 to 0.4.4 in /server Bumps me.champeau.gradle.japicmp from 0.4.3 to 0.4.4. --- updated-dependencies: - dependency-name: me.champeau.gradle.japicmp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + server/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84aae800bb50a..efb50db28a4a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) - Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) - Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) +- Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/server/build.gradle b/server/build.gradle index eccaf8a127647..c19e171c90f96 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -36,7 +36,7 @@ plugins { id('opensearch.publish') id('opensearch.internal-cluster-test') id('opensearch.optional-dependencies') - id('me.champeau.gradle.japicmp') version '0.4.3' + id('me.champeau.gradle.japicmp') version '0.4.4' } publishing { From 0ff0439dce988344c76ec0d68643bef528c652b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:41:55 -0400 Subject: [PATCH 231/357] Bump com.google.oauth-client:google-oauth-client from 1.35.0 to 1.36.0 in /plugins/discovery-gce (#16306) * Bump com.google.oauth-client:google-oauth-client Bumps [com.google.oauth-client:google-oauth-client](https://github.com/googleapis/google-oauth-java-client) from 1.35.0 to 1.36.0. - [Release notes](https://github.com/googleapis/google-oauth-java-client/releases) - [Changelog](https://github.com/googleapis/google-oauth-java-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-oauth-java-client/compare/v1.35.0...v1.36.0) --- updated-dependencies: - dependency-name: com.google.oauth-client:google-oauth-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/discovery-gce/build.gradle | 2 +- .../discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 | 1 - .../discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index efb50db28a4a4..a99c012bfc6d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) - Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) - Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) +- Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index a08fa1d968e30..76beb78bf533c 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -20,7 +20,7 @@ opensearchplugin { dependencies { api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0" api "com.google.api-client:google-api-client:1.35.2" - api "com.google.oauth-client:google-oauth-client:1.35.0" + api "com.google.oauth-client:google-oauth-client:1.36.0" api "com.google.http-client:google-http-client:${versions.google_http_client}" api "com.google.http-client:google-http-client-gson:${versions.google_http_client}" api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}" diff --git a/plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 b/plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 deleted file mode 100644 index a52e79088c7ca..0000000000000 --- a/plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2f52003156e40ba8be5f349a2716a77428896e69 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 b/plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 new file mode 100644 index 0000000000000..25aa7d76f153a --- /dev/null +++ b/plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1 @@ -0,0 +1 @@ +dc3f07bc8f49dd52fe8fcc15958f3cfeb003e20f \ No newline at end of file From 32c1a4370ce394fa9ae50835de626a24e1e81b2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:19:13 -0500 Subject: [PATCH 232/357] Bump lycheeverse/lychee-action from 1.10.0 to 2.0.2 (#16310) * Bump lycheeverse/lychee-action from 1.10.0 to 2.0.2 Bumps [lycheeverse/lychee-action](https://github.com/lycheeverse/lychee-action) from 1.10.0 to 2.0.2. - [Release notes](https://github.com/lycheeverse/lychee-action/releases) - [Commits](https://github.com/lycheeverse/lychee-action/compare/v1.10.0...v2.0.2) --- updated-dependencies: - dependency-name: lycheeverse/lychee-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/links.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 8f628fcd78148..cadbe71bb6ea8 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - name: lychee Link Checker id: lychee - uses: lycheeverse/lychee-action@v1.10.0 + uses: lycheeverse/lychee-action@v2.0.2 with: args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes fail: true diff --git a/CHANGELOG.md b/CHANGELOG.md index a99c012bfc6d2..1523d0496dace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) - Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) - Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306)) +- Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) From 931339e38be8f29281501a5ac8f0dddf2aa2232d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:28:05 -0400 Subject: [PATCH 233/357] Bump com.azure:azure-core-http-netty from 1.15.4 to 1.15.5 in /plugins/repository-azure (#16311) * Bump com.azure:azure-core-http-netty in /plugins/repository-azure Bumps [com.azure:azure-core-http-netty](https://github.com/Azure/azure-sdk-for-java) from 1.15.4 to 1.15.5. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core-http-netty_1.15.4...azure-core-http-netty_1.15.5) --- updated-dependencies: - dependency-name: com.azure:azure-core-http-netty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-core-http-netty-1.15.4.jar.sha1 | 1 - .../licenses/azure-core-http-netty-1.15.5.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1523d0496dace..4c279b27a156b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) - Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333)) -- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133)) +- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.5 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133), [#16311](https://github.com/opensearch-project/OpenSearch/pull/16311)) - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) - Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index e2d1c15cdb36f..d7eebe70ec303 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -48,7 +48,7 @@ dependencies { api 'com.azure:azure-json:1.3.0' api 'com.azure:azure-xml:1.1.0' api 'com.azure:azure-storage-common:12.25.1' - api 'com.azure:azure-core-http-netty:1.15.4' + api 'com.azure:azure-core-http-netty:1.15.5' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" api "io.netty:netty-codec-http2:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 deleted file mode 100644 index 97e6fad264294..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -489a38c9e6efb5ce01fbd276d8cb6c0e89000459 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1 new file mode 100644 index 0000000000000..2f5239cc26148 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1 @@ -0,0 +1 @@ +44d99705d3759e2ad7ee8110f811d4ed304a6a7c \ No newline at end of file From 783d3e1850e156161cb64519c2ab471290c3791a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:31:58 -0400 Subject: [PATCH 234/357] Bump com.google.code.gson:gson from 2.10.1 to 2.11.0 in /plugins/repository-gcs (#16308) * Bump com.google.code.gson:gson in /plugins/repository-gcs Bumps [com.google.code.gson:gson](https://github.com/google/gson) from 2.10.1 to 2.11.0. - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/main/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.10.1...gson-parent-2.11.0) --- updated-dependencies: - dependency-name: com.google.code.gson:gson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 | 1 - plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c279b27a156b..e60c078f5ed3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) - Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306)) - Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310)) +- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#16308](https://github.com/opensearch-project/OpenSearch/pull/16308)) ### Changed - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index ab129ab7f116a..b90bcc7f822d1 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -70,7 +70,7 @@ dependencies { api 'com.google.cloud:google-cloud-core-http:2.23.0' api 'com.google.cloud:google-cloud-storage:1.113.1' - api 'com.google.code.gson:gson:2.10.1' + api 'com.google.code.gson:gson:2.11.0' runtimeOnly "com.google.guava:guava:${versions.guava}" api 'com.google.guava:failureaccess:1.0.1' diff --git a/plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 deleted file mode 100644 index 9810309d1013a..0000000000000 --- a/plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3add478d4382b78ea20b1671390a858002feb6c \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 b/plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 new file mode 100644 index 0000000000000..0414a49526895 --- /dev/null +++ b/plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1 @@ -0,0 +1 @@ +527175ca6d81050b53bdd4c457a6d6e017626b0e \ No newline at end of file From 88d13eb5bd3bbea3a04b7b0c7576ca00d0f8435c Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:29:57 -0700 Subject: [PATCH 235/357] Enable coordinator search.request_stats_enabled by default (#16290) Signed-off-by: David Zane Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../java/org/opensearch/action/search/SearchRequestStats.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e60c078f5ed3a..00664189ced1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) - Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) +- Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290)) - Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) - Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java index d1d5f568fc09d..94200d29a4f21 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java @@ -32,7 +32,7 @@ public final class SearchRequestStats extends SearchRequestOperationsListener { public static final String SEARCH_REQUEST_STATS_ENABLED_KEY = "search.request_stats_enabled"; public static final Setting SEARCH_REQUEST_STATS_ENABLED = Setting.boolSetting( SEARCH_REQUEST_STATS_ENABLED_KEY, - false, + true, Setting.Property.Dynamic, Setting.Property.NodeScope ); From 9ddee61b1b4eafebe9b4d30e997b40178c939a5e Mon Sep 17 00:00:00 2001 From: kkewwei Date: Tue, 15 Oct 2024 04:54:01 +0800 Subject: [PATCH 236/357] Flat object field should delegate to keyword field for most query types (#14383) Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../test/index/90_flat_object.yml | 32 - .../92_flat_object_support_doc_values.yml | 788 +++++++++++++ .../xcontent/JsonToStringXContentParser.java | 8 +- .../index/mapper/FlatObjectFieldMapper.java | 310 +++-- .../index/mapper/KeywordFieldMapper.java | 62 +- .../mapper/FlatObjectFieldMapperTests.java | 45 +- .../mapper/FlatObjectFieldTypeTests.java | 1002 ++++++++++++++++- 8 files changed, 1990 insertions(+), 258 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 00664189ced1c..9665133e91207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) +- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index e8da81d7bee41..2a469aa5ff04d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -671,38 +671,6 @@ teardown: - match: { error.root_cause.0.reason: "Mapping definition for [data] has unsupported parameters: [analyzer : standard]"} - match: { status: 400 } - # Wildcard Query with dot path. - - do: - catch: bad_request - search: - body: { - _source: true, - query: { - "wildcard": { - "catalog.title": "Mock*" - } - } - } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog.title] which is of type [flat_object]"} - - match: { status: 400 } - - # Wildcard Query without dot path. - - do: - catch: bad_request - search: - body: { - _source: true, - query: { - "wildcard": { - "catalog": "Mock*" - } - } - } - - match: { error.root_cause.0.type: "query_shard_exception" } - - match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog] which is of type [flat_object]" } - - match: { status: 400 } - # Aggregation and Match Query with dot path. - do: catch: bad_request diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml new file mode 100644 index 0000000000000..9ec39660a4928 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml @@ -0,0 +1,788 @@ +--- +# The test setup includes: +# - Create flat_object mapping for flat_object_doc_values_test index +# - Index 9 example documents +# - Search tests about doc_values and index + +setup: + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0 " + + - do: + indices.create: + index: flat_object_doc_values_test + body: + mappings: + properties: + issue: + properties: + labels: + type: "flat_object" + order: + type: "keyword" + + - do: + bulk: + refresh: true + body: | + {"index":{"_index":"flat_object_doc_values_test","_id":"0"}} + {"order":"order0","issue":{"labels":{"number":1,"name":"abc0","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"1"}} + {"order":"order1","issue":{"labels":{"number":2,"name":"abc1","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"2"}} + {"order":"order2","issue":{"labels":{"number":2,"name":"abc2","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"3"}} + {"order":"order3","issue":{"labels":{"number":3,"name":"abc3","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"4"}} + {"order":"order4","issue":{"labels":{"number":4,"name":"abc4","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"5"}} + {"order":"order5","issue":{"labels":{"number":5,"name":"abc5","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"6"}} + {"order":"order6","issue":{"labels":{"number":6,"name":"abc6","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"7"}} + {"order":"order7","issue":{"labels":{"number":7,"name":"abc7","status":1}}} + {"index":{"_index":"flat_object_doc_values_test","_id":"8"}} + {"order":"order8","issue":{"labels":{"number":8,"name":"abc8","status":1}}} + +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: flat_object_doc_values_test + +--- +"Supported queries": + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0 " + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: { } + } + } + + - length: { hits.hits: 9 } + + # Term Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + term: { + issue.labels.status: 1 + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + issue.labels.name: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Term Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + term: { + issue.labels: 1 + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + issue.labels: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Terms Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + terms: { + issue.labels.status: [0,1] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + terms: { + issue.labels.name: ["abc8"] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Terms Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + terms: { + issue.labels: [ 0,1 ] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + terms: { + issue.labels.name: ["abc8"] + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Prefix Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + prefix: { + issue.labels.name: "ab" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + prefix: { + issue.labels.name: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Prefix Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + prefix: { + issue.labels: "ab" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + prefix: { + issue.labels: "abc8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Regexp Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + regexp: { + issue.labels.name: "ab.*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + - match: { hits.hits.0._source.issue.labels.name: "abc8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + regexp: { + issue.labels.name: "a.*c8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Regexp Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + regexp: { + issue.labels: "ab.*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + regexp: { + issue.labels: "a.*c8" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Fuzzy Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + fuzzy: { + issue.labels.name: { + value: "abcx", + fuzziness: 1 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + fuzzy: { + issue.labels.name: { + value: "abc8", + fuzziness: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Fuzzy Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + fuzzy: { + issue.labels: { + value: "abcx", + fuzziness: 1 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + fuzzy: { + issue.labels: { + value: "abc8", + fuzziness: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Range Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + range: { + issue.labels.status: { + from: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + range: { + issue.labels.name: { + from: "abc8" + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Range Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + range: { + issue.labels: { + from: 0 + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + range: { + issue.labels: { + from: "abc8" + } + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Exists Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + exists: { + field: "issue.labels.status" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Exists Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + exists: { + field: "issue.labels" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + + # Wildcard Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + wildcard: { + issue.labels.name: "abc*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + wildcard: { + issue.labels.name: "abc8*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + # Wildcard Query with no path. + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + term: { + order: "order8" + } + }, + { + wildcard: { + issue.labels: "abc*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } + + - do: + search: + body: { + _source: true, + query: { + bool: { + must: [ + { + wildcard: { + issue.labels: "abc8*" + } + } + ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.order: "order8" } diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 95a8d9c9495f2..21270b4241b15 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -37,6 +37,10 @@ * @opensearch.internal */ public class JsonToStringXContentParser extends AbstractXContentParser { + public static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + public static final String VALUE_SUFFIX = "._value"; + public static final String DOT_SYMBOL = "."; + public static final String EQUAL_SYMBOL = "="; private final String fieldTypeName; private final XContentParser parser; @@ -50,10 +54,6 @@ public class JsonToStringXContentParser extends AbstractXContentParser { private final DeprecationHandler deprecationHandler; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - private static final String EQUAL_SYMBOL = "="; - public JsonToStringXContentParser( NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 738efcfafdca1..0ccdb40f9d33a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -16,19 +16,16 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; -import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; -import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.lucene.search.AutomatonQueries; +import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.JsonToStringXContentParser; import org.opensearch.core.common.ParsingException; import org.opensearch.core.xcontent.DeprecationHandler; @@ -37,8 +34,8 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.query.QueryShardException; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -52,7 +49,11 @@ import java.util.function.BiFunction; import java.util.function.Supplier; -import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.DOT_SYMBOL; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.EQUAL_SYMBOL; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.FlatObjectFieldType.getKeywordFieldType; /** * A field mapper for flat_objects. @@ -62,10 +63,6 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { public static final String CONTENT_TYPE = "flat_object"; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - private static final String DOT_SYMBOL = "."; - private static final String EQUAL_SYMBOL = "="; /** * In flat_object field mapper, field type is similar to keyword field type @@ -86,7 +83,12 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name()); + return new FlatObjectFieldType( + this.name() + DOT_SYMBOL + key, + this.name(), + (KeywordFieldType) valueFieldMapper.fieldType(), + (KeywordFieldType) valueAndPathFieldMapper.fieldType() + ); } /** @@ -111,20 +113,12 @@ public Builder(String name) { builder = this; } - private FlatObjectFieldType buildFlatObjectFieldType(BuilderContext context, FieldType fieldType) { - return new FlatObjectFieldType(buildFullName(context), fieldType); - } - /** * ValueFieldMapper is the subfield type for values in the Json. * use a {@link KeywordFieldMapper.KeywordField} */ - private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { - String fullName = buildFullName(context); + private ValueFieldMapper buildValueFieldMapper(FieldType fieldType, KeywordFieldType valueFieldType) { FieldType vft = new FieldType(fieldType); - KeywordFieldMapper.KeywordFieldType valueFieldType = new KeywordFieldMapper.KeywordFieldType(fullName + VALUE_SUFFIX, vft); - - fft.setValueFieldType(valueFieldType); return new ValueFieldMapper(vft, valueFieldType); } @@ -132,27 +126,30 @@ private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType * ValueAndPathFieldMapper is the subfield type for path=value format in the Json. * also use a {@link KeywordFieldMapper.KeywordField} */ - private ValueAndPathFieldMapper buildValueAndPathFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { - String fullName = buildFullName(context); + private ValueAndPathFieldMapper buildValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType valueAndPathFieldType) { FieldType vft = new FieldType(fieldType); - KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType = new KeywordFieldMapper.KeywordFieldType( - fullName + VALUE_AND_PATH_SUFFIX, - vft - ); - fft.setValueAndPathFieldType(ValueAndPathFieldType); - return new ValueAndPathFieldMapper(vft, ValueAndPathFieldType); + return new ValueAndPathFieldMapper(vft, valueAndPathFieldType); } @Override public FlatObjectFieldMapper build(BuilderContext context) { - FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); - FlatObjectFieldType fft = buildFlatObjectFieldType(context, fieldtype); + boolean isSearchable = true; + boolean hasDocValue = true; + KeywordFieldType valueFieldType = getKeywordFieldType(buildFullName(context), VALUE_SUFFIX, isSearchable, hasDocValue); + KeywordFieldType valueAndPathFieldType = getKeywordFieldType( + buildFullName(context), + VALUE_AND_PATH_SUFFIX, + isSearchable, + hasDocValue + ); + FlatObjectFieldType fft = new FlatObjectFieldType(buildFullName(context), null, valueFieldType, valueAndPathFieldType); + return new FlatObjectFieldMapper( name, Defaults.FIELD_TYPE, fft, - buildValueFieldMapper(context, fieldtype, fft), - buildValueAndPathFieldMapper(context, fieldtype, fft), + buildValueFieldMapper(Defaults.FIELD_TYPE, valueFieldType), + buildValueAndPathFieldMapper(Defaults.FIELD_TYPE, valueAndPathFieldType), CopyTo.empty(), this ); @@ -189,66 +186,70 @@ public static final class FlatObjectFieldType extends StringFieldType { private final String mappedFieldTypeName; - private KeywordFieldMapper.KeywordFieldType valueFieldType; + private final KeywordFieldType valueFieldType; - private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; - - public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { - super(name, isSearchable, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); - this.ignoreAbove = Integer.MAX_VALUE; - this.nullValue = null; - this.mappedFieldTypeName = null; - } + private final KeywordFieldType valueAndPathFieldType; - public FlatObjectFieldType(String name, FieldType fieldType) { + public FlatObjectFieldType( + String name, + String mappedFieldTypeName, + boolean isSearchable, + boolean hasDocValues, + NamedAnalyzer analyzer, + Map meta + ) { super( name, - fieldType.indexOptions() != IndexOptions.NONE, + isSearchable, false, - true, - new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), - Collections.emptyMap() + hasDocValues, + analyzer == null ? TextSearchInfo.SIMPLE_MATCH_ONLY : new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), + meta ); + setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - this.mappedFieldTypeName = null; - } - - public FlatObjectFieldType(String name, NamedAnalyzer analyzer) { - super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); - this.ignoreAbove = Integer.MAX_VALUE; - this.nullValue = null; - this.mappedFieldTypeName = null; + this.mappedFieldTypeName = mappedFieldTypeName; + this.valueFieldType = getKeywordFieldType(name, VALUE_SUFFIX, isSearchable, hasDocValues); + this.valueAndPathFieldType = getKeywordFieldType(name, VALUE_AND_PATH_SUFFIX, isSearchable, hasDocValues); } - public FlatObjectFieldType(String name, String mappedFieldTypeName) { + public FlatObjectFieldType( + String name, + String mappedFieldTypeName, + KeywordFieldType valueFieldType, + KeywordFieldType valueAndPathFieldType + ) { super( name, - true, + valueFieldType.isSearchable(), false, - true, + valueFieldType.hasDocValues(), new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.mappedFieldTypeName = mappedFieldTypeName; - } - - void setValueFieldType(KeywordFieldMapper.KeywordFieldType valueFieldType) { this.valueFieldType = valueFieldType; + this.valueAndPathFieldType = valueAndPathFieldType; } - void setValueAndPathFieldType(KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType) { - this.valueAndPathFieldType = ValueAndPathFieldType; + static KeywordFieldType getKeywordFieldType(String fullName, String valueType, boolean isSearchable, boolean hasDocValue) { + return new KeywordFieldType(fullName + valueType, isSearchable, hasDocValue, Collections.emptyMap()) { + @Override + protected String rewriteForDocValue(Object value) { + assert value instanceof String; + return fullName + DOT_SYMBOL + value; + } + }; } - public KeywordFieldMapper.KeywordFieldType getValueFieldType() { + public KeywordFieldType getValueFieldType() { return this.valueFieldType; } - public KeywordFieldMapper.KeywordFieldType getValueAndPathFieldType() { + public KeywordFieldType getValueAndPathFieldType() { return this.valueAndPathFieldType; } @@ -331,6 +332,10 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } + private KeywordFieldType valueFieldType() { + return (mappedFieldTypeName == null) ? valueFieldType : valueAndPathFieldType; + } + /** * redirect queries with rewrite value to rewriteSearchValue and directSubFieldName */ @@ -352,17 +357,12 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { @Override public Query termsQuery(List values, QueryShardContext context) { - failIfNotIndexed(); - String directedSearchFieldName = directSubfield(); - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - String rewriteValues = rewriteValue(inputToString(values.get(i))); - - bytesRefs[i] = indexedValueForSearch(new BytesRef(rewriteValues)); - + List parsedValues = new ArrayList<>(values.size()); + for (Object value : values) { + parsedValues.add(rewriteValue(inputToString(value))); } - return new TermInSetQuery(directedSearchFieldName, bytesRefs); + return valueFieldType().termsQuery(parsedValues, context); } /** @@ -395,7 +395,7 @@ public String rewriteValue(String searchValueString) { } - private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { + boolean hasMappedFieldTyeNameInQueryFieldName(String input) { String prefix = this.mappedFieldTypeName; if (prefix == null) { return false; @@ -413,6 +413,9 @@ private boolean hasMappedFieldTyeNameInQueryFieldName(String input) { } private String inputToString(Object inputValue) { + if (inputValue == null) { + return null; + } if (inputValue instanceof Integer) { String inputToString = Integer.toString((Integer) inputValue); return inputToString; @@ -448,46 +451,50 @@ private String inputToString(Object inputValue) { @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { - String directSubfield = directSubfield(); - String rewriteValue = rewriteValue(value); - - if (context.allowExpensiveQueries() == false) { - throw new OpenSearchException( - "[prefix] queries cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false. For optimised prefix queries on text " - + "fields please enable [index_prefixes]." - ); - } - failIfNotIndexed(); - if (method == null) { - method = MultiTermQuery.CONSTANT_SCORE_REWRITE; - } - if (caseInsensitive) { - return AutomatonQueries.caseInsensitivePrefixQuery((new Term(directSubfield, indexedValueForSearch(rewriteValue))), method); - } - return new PrefixQuery(new Term(directSubfield, indexedValueForSearch(rewriteValue)), method); + return valueFieldType().prefixQuery(rewriteValue(value), method, caseInsensitive, context); + } + + @Override + public Query regexpQuery( + String value, + int syntaxFlags, + int matchFlags, + int maxDeterminizedStates, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + return valueFieldType().regexpQuery(rewriteValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + } + + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + return valueFieldType().fuzzyQuery( + rewriteValue(inputToString(value)), + fuzziness, + prefixLength, + maxExpansions, + transpositions, + method, + context + ); } @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - String directSubfield = directSubfield(); - String rewriteUpperTerm = rewriteValue(inputToString(upperTerm)); - String rewriteLowerTerm = rewriteValue(inputToString(lowerTerm)); - if (context.allowExpensiveQueries() == false) { - throw new OpenSearchException( - "[range] queries on [text] or [keyword] fields cannot be executed when '" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "' is set to false." - ); - } - failIfNotIndexed(); - return new TermRangeQuery( - directSubfield, - lowerTerm == null ? null : indexedValueForSearch(rewriteLowerTerm), - upperTerm == null ? null : indexedValueForSearch(rewriteUpperTerm), + return valueFieldType().rangeQuery( + rewriteValue(inputToString(lowerTerm)), + rewriteValue(inputToString(upperTerm)), includeLower, - includeUpper + includeUpper, + context ); } @@ -503,8 +510,12 @@ public Query existsQuery(QueryShardContext context) { searchKey = this.mappedFieldTypeName; searchField = name(); } else { - searchKey = FieldNamesFieldMapper.NAME; - searchField = name(); + if (hasDocValues()) { + return new FieldExistsQuery(name()); + } else { + searchKey = FieldNamesFieldMapper.NAME; + searchField = name(); + } } return new TermQuery(new Term(searchKey, indexedValueForSearch(searchField))); } @@ -516,13 +527,7 @@ public Query wildcardQuery( boolean caseInsensitve, QueryShardContext context ) { - // flat_object field types are always normalized, so ignore case sensitivity and force normalize the wildcard - // query text - throw new QueryShardException( - context, - "Can only use wildcard queries on keyword and text fields - not on [" + name() + "] which is of type [" + typeName() + "]" - ); - + return valueFieldType().wildcardQuery(rewriteValue(value), method, caseInsensitve, context); } } @@ -606,7 +611,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } } - } @Override @@ -637,6 +641,8 @@ public Iterator iterator() { */ private void parseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { + assert valueFieldMapper != null; + assert valueAndPathFieldMapper != null; NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { value = normalizeValue(normalizer, name(), value); @@ -647,69 +653,57 @@ private void parseValueAddFields(ParseContext context, String value, String fiel if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(fieldType().name() + DOT_SYMBOL + value); - Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } if (fieldName.equals(fieldType().name())) { + Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - } - if (valueType.equals(VALUE_SUFFIX)) { - if (valueFieldMapper != null) { - valueFieldMapper.addField(context, value); - } - } - if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - if (valueAndPathFieldMapper != null) { - valueAndPathFieldMapper.addField(context, value); - } + } else if (valueType.equals(VALUE_SUFFIX)) { + valueFieldMapper.addField(context, value); + } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + valueAndPathFieldMapper.addField(context, value); } if (fieldType().hasDocValues()) { if (fieldName.equals(fieldType().name())) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); - } - if (valueType.equals(VALUE_SUFFIX)) { - if (valueFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); - } - } - if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { - if (valueAndPathFieldMapper != null) { - context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); - } + } else if (valueType.equals(VALUE_SUFFIX)) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_SUFFIX, binaryValue)); + } else if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { + context.doc().add(new SortedSetDocValuesField(fieldType().name() + VALUE_AND_PATH_SUFFIX, binaryValue)); } } - } - } private static String normalizeValue(NamedAnalyzer normalizer, String field, String value) throws IOException { - String normalizerErrorMessage = "The normalization token stream is " - + "expected to produce exactly 1 token, but got 0 for analyzer " - + normalizer - + " and input \"" - + value - + "\""; try (TokenStream ts = normalizer.tokenStream(field, value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); if (ts.incrementToken() == false) { - throw new IllegalStateException(normalizerErrorMessage); + throw new IllegalStateException(errorMessage(normalizer, value)); } final String newValue = termAtt.toString(); if (ts.incrementToken()) { - throw new IllegalStateException(normalizerErrorMessage); + throw new IllegalStateException(errorMessage(normalizer, value)); } ts.end(); return newValue; } } + private static String errorMessage(NamedAnalyzer normalizer, String value) { + return "The normalization token stream is " + + "expected to produce exactly 1 token, but got 0 for analyzer " + + normalizer + + " and input \"" + + value + + "\""; + } + @Override protected String contentType() { return CONTENT_TYPE; @@ -717,7 +711,7 @@ protected String contentType() { private static final class ValueAndPathFieldMapper extends FieldMapper { - protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { + protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -728,7 +722,7 @@ void addField(ParseContext context, String value) { context.doc().add(field); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } } @@ -758,7 +752,7 @@ public String toString() { private static final class ValueFieldMapper extends FieldMapper { - protected ValueFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { + protected ValueFieldMapper(FieldType fieldType, KeywordFieldType mappedFieldType) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } @@ -768,7 +762,7 @@ void addField(ParseContext context, String value) { Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); context.doc().add(field); - if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { + if (fieldType().hasDocValues() == false) { createFieldNamesField(context); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 11ff601b3fd6d..54a1aead5fcc7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -263,7 +263,7 @@ public KeywordFieldMapper build(BuilderContext context) { * * @opensearch.internal */ - public static final class KeywordFieldType extends StringFieldType { + public static class KeywordFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; @@ -387,6 +387,10 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } + protected Object rewriteForDocValue(Object value) { + return value; + } + @Override public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); @@ -395,19 +399,21 @@ public Query termsQuery(List values, QueryShardContext context) { if (!context.keywordFieldIndexOrDocValuesEnabled()) { return super.termsQuery(values, context); } - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); + BytesRef[] iBytesRefs = new BytesRef[values.size()]; + BytesRef[] dVByteRefs = new BytesRef[values.size()]; + for (int i = 0; i < iBytesRefs.length; i++) { + iBytesRefs[i] = indexedValueForSearch(values.get(i)); + dVByteRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); } - Query indexQuery = new TermInSetQuery(name(), bytesRefs); - Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + Query indexQuery = new TermInSetQuery(name(), iBytesRefs); + Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), dVByteRefs); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } // if we only have doc_values enabled, we construct a new query with doc_values re-written if (hasDocValues()) { BytesRef[] bytesRefs = new BytesRef[values.size()]; for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); + bytesRefs[i] = indexedValueForSearch(rewriteForDocValue(values.get(i))); } return new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); } @@ -436,17 +442,25 @@ public Query prefixQuery( return super.prefixQuery(value, method, caseInsensitive, context); } Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); - Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); + Query dvQuery = super.prefixQuery( + (String) rewriteForDocValue(value), + MultiTermQuery.DOC_VALUES_REWRITE, + caseInsensitive, + context + ); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { if (caseInsensitive) { return AutomatonQueries.caseInsensitivePrefixQuery( - (new Term(name(), indexedValueForSearch(value))), + (new Term(name(), indexedValueForSearch(rewriteForDocValue(value)))), MultiTermQuery.DOC_VALUES_REWRITE ); } - return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); + return new PrefixQuery( + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), + MultiTermQuery.DOC_VALUES_REWRITE + ); } return super.prefixQuery(value, method, caseInsensitive, context); } @@ -472,7 +486,7 @@ public Query regexpQuery( } Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); Query dvQuery = super.regexpQuery( - value, + (String) rewriteForDocValue(value), syntaxFlags, matchFlags, maxDeterminizedStates, @@ -483,7 +497,7 @@ public Query regexpQuery( } if (hasDocValues()) { return new RegexpQuery( - new Term(name(), indexedValueForSearch(value)), + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), syntaxFlags, matchFlags, RegexpQuery.DEFAULT_PROVIDER, @@ -514,8 +528,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower ); Query dvQuery = new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), + lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), + upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -525,8 +539,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower if (hasDocValues()) { return new TermRangeQuery( name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), + lowerTerm == null ? null : indexedValueForSearch(rewriteForDocValue(lowerTerm)), + upperTerm == null ? null : indexedValueForSearch(rewriteForDocValue(upperTerm)), includeLower, includeUpper, MultiTermQuery.DOC_VALUES_REWRITE @@ -563,7 +577,7 @@ public Query fuzzyQuery( } Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); Query dvQuery = super.fuzzyQuery( - value, + rewriteForDocValue(value), fuzziness, prefixLength, maxExpansions, @@ -575,8 +589,8 @@ public Query fuzzyQuery( } if (hasDocValues()) { return new FuzzyQuery( - new Term(name(), indexedValueForSearch(value)), - fuzziness.asDistance(BytesRefs.toString(value)), + new Term(name(), indexedValueForSearch(rewriteForDocValue(value))), + fuzziness.asDistance(BytesRefs.toString(rewriteForDocValue(value))), prefixLength, maxExpansions, transpositions, @@ -607,13 +621,19 @@ public Query wildcardQuery( return super.wildcardQuery(value, method, caseInsensitive, true, context); } Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); - Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); + Query dvQuery = super.wildcardQuery( + (String) rewriteForDocValue(value), + MultiTermQuery.DOC_VALUES_REWRITE, + caseInsensitive, + true, + context + ); return new IndexOrDocValuesQuery(indexQuery, dvQuery); } if (hasDocValues()) { Term term; value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); - term = new Term(name(), value); + term = new Term(name(), (String) rewriteForDocValue(value)); if (caseInsensitive) { return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); } diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index 94d1f501bee51..afd9e994ce3ae 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; @@ -23,14 +24,13 @@ import java.io.IOException; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.IsEqual.equalTo; public class FlatObjectFieldMapperTests extends MapperTestCase { - private static final String FIELD_TYPE = "flat_object"; - private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; - private static final String VALUE_SUFFIX = "._value"; - protected boolean supportsMeta() { return false; } @@ -41,7 +41,7 @@ protected boolean supportsOrIgnoresBoost() { public void testMapperServiceHasParser() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); })); - Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(FIELD_TYPE); + Mapper.TypeParser parser = mapperService.mapperRegistry.getMapperParsers().get(CONTENT_TYPE); assertNotNull(parser); assertTrue(parser instanceof FlatObjectFieldMapper.TypeParser); } @@ -49,28 +49,39 @@ public void testMapperServiceHasParser() throws IOException { protected void assertExistsQuery(MapperService mapperService) throws IOException { ParseContext.Document fields = mapperService.documentMapper().parse(source(this::writeField)).rootDoc(); QueryShardContext queryShardContext = createQueryShardContext(mapperService); - MappedFieldType fieldType = mapperService.fieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType fieldType = (FlatObjectFieldMapper.FlatObjectFieldType) mapperService.fieldType("field"); Query query = fieldType.existsQuery(queryShardContext); assertExistsQuery(fieldType, query, fields); - } - protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) { - // we always perform a term query against _field_names, even when the field - // is not added to _field_names because it is not indexed nor stored - assertThat(query, instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) query; - assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); - assertEquals("field", termQuery.getTerm().text()); - if (fieldType.isSearchable() || fieldType.isStored()) { - assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); + protected void assertExistsQuery(FlatObjectFieldMapper.FlatObjectFieldType fieldType, Query query, ParseContext.Document fields) { + + if (fieldType.hasDocValues() && fieldType.hasMappedFieldTyeNameInQueryFieldName(fieldType.name()) == false) { + assertThat(query, instanceOf(FieldExistsQuery.class)); + FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; + assertEquals(fieldType.name(), fieldExistsQuery.getField()); } else { + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field()); + assertEquals("field", termQuery.getTerm().text()); + } + + if (fieldType.hasDocValues()) { + assertDocValuesField(fields, "field"); assertNoFieldNamesField(fields); + } else { + assertNoDocValuesField(fields, "field"); + if (fieldType.isSearchable()) { + assertNotNull(fields.getField(FieldNamesFieldMapper.NAME)); + } else { + assertNoFieldNamesField(fields); + } } } public void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", FIELD_TYPE); + b.field("type", CONTENT_TYPE); } /** diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java index 9ec053dc59d10..38a6f13777f00 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -8,11 +8,23 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.FieldExistsQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; -import org.opensearch.Version; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Operations; +import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.analysis.AnalyzerScope; import org.opensearch.index.analysis.NamedAnalyzer; @@ -21,18 +33,41 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_AND_PATH_SUFFIX; +import static org.opensearch.common.xcontent.JsonToStringXContentParser.VALUE_SUFFIX; +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; +import static org.apache.lucene.search.MultiTermQuery.DOC_VALUES_REWRITE; + public class FlatObjectFieldTypeTests extends FieldTypeTestCase { - private static MappedFieldType getFlatParentFieldType(String fieldName) { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); - Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - MappedFieldType flatParentFieldType = new FlatObjectFieldMapper.Builder(fieldName).build(context).fieldType(); - return flatParentFieldType; + + private static MappedFieldType getFlatParentFieldType( + String fieldName, + String mappedFieldTypeName, + boolean isSearchable, + boolean hasDocValues + ) { + FlatObjectFieldMapper.Builder builder = new FlatObjectFieldMapper.Builder(fieldName); + FlatObjectFieldMapper.FlatObjectFieldType flatObjectFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + fieldName, + mappedFieldTypeName, + isSearchable, + hasDocValues, + null, + Collections.emptyMap() + ); + FieldType fieldtype = new FieldType(FlatObjectFieldMapper.Defaults.FIELD_TYPE); + FieldType vft = new FieldType(fieldtype); + if (flatObjectFieldType.isSearchable() == false) { + vft.setIndexOptions(IndexOptions.NONE); + } + return flatObjectFieldType; } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = getFlatParentFieldType("field"); + MappedFieldType mapper = getFlatParentFieldType("field", null, true, true); Map jsonPoint = new HashMap<>(); jsonPoint.put("type", "flat_object"); @@ -54,32 +89,48 @@ public void testFetchSourceValue() throws IOException { public void testDirectSubfield() { { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = + (FlatObjectFieldMapper.FlatObjectFieldType) (getFlatParentFieldType("field", null, true, true)); // when searching for "foo" in "field", the directSubfield is field._value field - String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); + String searchFieldName = (flatParentFieldType).directSubfield(); assertEquals("field._value", searchFieldName); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the directSubfield is field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); assertEquals("field._valueAndPath", searchFieldNameDocPath); } { NamedAnalyzer analyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, null); - MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", analyzer); + MappedFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType("field", null, true, true, analyzer, Collections.emptyMap()); assertEquals("field._value", ((FlatObjectFieldMapper.FlatObjectFieldType) ft).directSubfield()); } } public void testRewriteValue() { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when searching for "foo" in "field", the rewrite value is "foo" - String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + String searchValues = (flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the rewrite value is "field.bar=foo" String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -89,15 +140,25 @@ public void testRewriteValue() { public void testTermQuery() { - MappedFieldType flatParentFieldType = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType flatParentFieldType = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when searching for "foo" in "field", the term query is directed to search "foo" in field._value field - String searchFieldName = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).directSubfield(); - String searchValues = ((FlatObjectFieldMapper.FlatObjectFieldType) flatParentFieldType).rewriteValue("foo"); + String searchFieldName = (flatParentFieldType).directSubfield(); + String searchValues = (flatParentFieldType).rewriteValue("foo"); assertEquals("foo", searchValues); assertEquals(new TermQuery(new Term(searchFieldName, searchValues)), flatParentFieldType.termQuery(searchValues, null)); - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", flatParentFieldType.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + flatParentFieldType.name(), + flatParentFieldType.getValueFieldType(), + flatParentFieldType.getValueAndPathFieldType() + ); // when searching for "foo" in "field.bar", the term query is directed to search in field._valueAndPath field String searchFieldNameDocPath = ((FlatObjectFieldMapper.FlatObjectFieldType) dynamicMappedFieldType).directSubfield(); @@ -105,30 +166,919 @@ public void testTermQuery() { assertEquals("field.bar=foo", searchValuesDocPath); assertEquals(new TermQuery(new Term(searchFieldNameDocPath, searchValuesDocPath)), dynamicMappedFieldType.termQuery("foo", null)); - MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType("field", false, true, Collections.emptyMap()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null)); + MappedFieldType unsearchable = new FlatObjectFieldMapper.FlatObjectFieldType( + "field", + null, + false, + true, + null, + Collections.emptyMap() + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> unsearchable.termQuery("bar", MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { { - MappedFieldType ft = getFlatParentFieldType("field"); + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); // when checking on the flat_object field name "field", check if exist in the field mapper names - assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + assertEquals(new FieldExistsQuery("field"), ft.existsQuery(null)); // when checking if a subfield within the flat_object, for example, "field.bar", use term query in the flat_object field - MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType("field.bar", ft.name()); + MappedFieldType dynamicMappedFieldType = new FlatObjectFieldMapper.FlatObjectFieldType( + "field.bar", + ft.name(), + ft.getValueFieldType(), + ft.getValueAndPathFieldType() + ); assertEquals(new TermQuery(new Term("field", "field.bar")), dynamicMappedFieldType.existsQuery(null)); } + { FlatObjectFieldMapper.FlatObjectFieldType ft = new FlatObjectFieldMapper.FlatObjectFieldType( "field", + null, true, false, + null, Collections.emptyMap() ); - assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null)); + assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + } + + public void testTermsQuery() { + + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new IndexOrDocValuesQuery( + new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms), + new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms) + ); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new IndexOrDocValuesQuery( + new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms), + new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms) + ); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + Query expected = new TermInSetQuery("field" + VALUE_SUFFIX, indexTerms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + Query expected = new TermInSetQuery("field" + VALUE_AND_PATH_SUFFIX, indexTerms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_SUFFIX, docValueterms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + + List indexTerms = new ArrayList<>(); + indexTerms.add(new BytesRef("foo")); + indexTerms.add(new BytesRef("bar")); + List docValueterms = new ArrayList<>(); + docValueterms.add(new BytesRef("field.foo")); + docValueterms.add(new BytesRef("field.bar")); + Query expected = new TermInSetQuery(DOC_VALUES_REWRITE, "field" + VALUE_AND_PATH_SUFFIX, docValueterms); + + assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testPrefixQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), + new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE), + new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), CONSTANT_SCORE_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new PrefixQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), DOC_VALUES_REWRITE); + assertEquals(expected, ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.prefixQuery("foo", CONSTANT_SCORE_REWRITE, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testRegexpQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ), + new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ), + new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new RegexpQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("field.foo")), + 0, + 0, + RegexpQuery.DEFAULT_PROVIDER, + 10, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.regexpQuery("foo", 0, 0, 10, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testFuzzyQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true), + new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true), + new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new FuzzyQuery(new Term("field" + VALUE_SUFFIX, "foo"), 2, 1, 50, true); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new FuzzyQuery(new Term("field" + VALUE_AND_PATH_SUFFIX, "foo"), 2, 1, 50, true); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new FuzzyQuery( + new Term("field" + VALUE_SUFFIX, "field.foo"), + 2, + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new FuzzyQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo"), + 2, + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testRangeQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), + new TermRangeQuery( + "field" + VALUE_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true), + new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new TermRangeQuery("field" + VALUE_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new TermRangeQuery("field" + VALUE_AND_PATH_SUFFIX, new BytesRef("2"), new BytesRef("10"), true, true); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new TermRangeQuery( + "field" + VALUE_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new TermRangeQuery( + "field" + VALUE_AND_PATH_SUFFIX, + new BytesRef("field.2"), + new BytesRef("field.10"), + true, + true, + DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(new BytesRef("2"), new BytesRef("10"), true, true, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + } + + public void testWildcardQuery() { + // 1.test isSearchable=true, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + true + ); + Query expected = new IndexOrDocValuesQuery( + new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ) + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 2.test isSearchable=true, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + true, + false + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=true, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + true, + false + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 3.test isSearchable=false, hasDocValues=true, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + true + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // test isSearchable=false, hasDocValues=true, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + true + ); + Query expected = new WildcardQuery( + new Term("field" + VALUE_AND_PATH_SUFFIX, "field.foo*"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.DOC_VALUES_REWRITE + ); + assertEquals(expected, ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); + } + + // 4.test isSearchable=false, hasDocValues=false, mappedFieldTypeName=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + null, + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._value] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); + } + + // test isSearchable=false, hasDocValues=false, mappedFieldTypeName!=null + { + FlatObjectFieldMapper.FlatObjectFieldType ft = (FlatObjectFieldMapper.FlatObjectFieldType) getFlatParentFieldType( + "field", + "field", + false, + false + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.wildcardQuery("foo*", null, false, MOCK_QSC_ENABLE_INDEX_DOC_VALUES) + ); + assertEquals( + "Cannot search on field [field._valueAndPath] since it is both not indexed, and does not have doc_values " + "enabled.", + e.getMessage() + ); } } } From 6c17119e1051aa9db339454df32aec0c18c52499 Mon Sep 17 00:00:00 2001 From: Michael Froh Date: Mon, 14 Oct 2024 16:41:17 -0700 Subject: [PATCH 237/357] Add new benchmark config for query approximation (#16323) We have guarded the experimental query approximation framework behind a feature flag. In order to easily measure the impact of approximation on big5 benchmarks, it would be nice to have a benchmark config. Signed-off-by: Michael Froh --- .github/benchmark-configs.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/benchmark-configs.json b/.github/benchmark-configs.json index 27b7228e1203a..732f2f9b96ae3 100644 --- a/.github/benchmark-configs.json +++ b/.github/benchmark-configs.json @@ -221,5 +221,23 @@ "data_instance_config": "4vCPU, 32G Mem, 16G Heap" }, "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline" + }, + "id_14": { + "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.", + "supported_major_versions": ["3"], + "cluster-benchmark-configs": { + "SINGLE_NODE_CLUSTER": "true", + "MIN_DISTRIBUTION": "true", + "TEST_WORKLOAD": "big5", + "ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true", + "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}", + "CAPTURE_NODE_STAT": "true", + "TEST_PROCEDURE": "restore-from-snapshot" + }, + "cluster_configuration": { + "size": "Single-Node", + "data_instance_config": "4vCPU, 32G Mem, 16G Heap" + }, + "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline" } } From a53e0c63aa9c6a85fed30e7bdce8b533aa471060 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:21:12 +0530 Subject: [PATCH 238/357] Update last seen cluster state in commit phase (#16215) * Update last seen cluster state on apply commit Signed-off-by: Sooraj Sinha --- CHANGELOG.md | 1 + .../opensearch/cluster/coordination/Coordinator.java | 8 +++++++- .../coordination/PublicationTransportHandler.java | 10 +++++++--- .../coordination/PublicationTransportHandlerTests.java | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9665133e91207..668846cc1da6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290)) - Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) - Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) +- Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 1b3ae89251ac0..02d5f8431f0ad 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -105,6 +105,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -383,7 +384,11 @@ void onFollowerCheckRequest(FollowerCheckRequest followerCheckRequest) { } } - private void handleApplyCommit(ApplyCommitRequest applyCommitRequest, ActionListener applyListener) { + private void handleApplyCommit( + ApplyCommitRequest applyCommitRequest, + Consumer updateLastSeen, + ActionListener applyListener + ) { synchronized (mutex) { logger.trace("handleApplyCommit: applying commit {}", applyCommitRequest); @@ -391,6 +396,7 @@ private void handleApplyCommit(ApplyCommitRequest applyCommitRequest, ActionList final ClusterState committedState = hideStateIfNotRecovered(coordinationState.get().getLastAcceptedState()); applierState = mode == Mode.CANDIDATE ? clusterStateWithNoClusterManagerBlock(committedState) : committedState; clusterApplier.setPreCommitState(applierState); + updateLastSeen.accept(coordinationState.get().getLastAcceptedState()); if (applyCommitRequest.getSourceNode().equals(getLocalNode())) { // cluster-manager node applies the committed state at the end of the publication process, not here. diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 42aa55433dd5f..d30efde52bffb 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -43,6 +43,7 @@ import org.opensearch.cluster.coordination.PersistedStateRegistry.PersistedStateType; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.TriConsumer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -65,7 +66,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -110,7 +110,7 @@ public PublicationTransportHandler( TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, Function handlePublishRequest, - BiConsumer> handleApplyCommit, + TriConsumer, ActionListener> handleApplyCommit, RemoteClusterStateService remoteClusterStateService ) { this.transportService = transportService; @@ -142,7 +142,7 @@ public PublicationTransportHandler( false, false, ApplyCommitRequest::new, - (request, channel, task) -> handleApplyCommit.accept(request, transportCommitCallback(channel)) + (request, channel, task) -> handleApplyCommit.apply(request, this::updateLastSeen, transportCommitCallback(channel)) ); } @@ -377,6 +377,10 @@ private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes dis return true; } + private void updateLastSeen(final ClusterState clusterState) { + lastSeenClusterState.set(clusterState); + } + // package private for testing void setCurrentPublishRequestToSelf(PublishRequest publishRequest) { this.currentPublishRequestToSelf.set(publishRequest); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 266928c919fe2..616559e91536d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -466,7 +466,7 @@ private PublicationTransportHandler getPublicationTransportHandler( transportService, writableRegistry(), handlePublishRequest, - (pu, l) -> {}, + (pu, uc, l) -> {}, remoteClusterStateService ); transportService.start(); From 35c366ddc794e0600184cf406c06ae65061e28ce Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:20:39 +0530 Subject: [PATCH 239/357] Add support to dynamically resize threadpools size (#16236) Signed-off-by: Gaurav Bafna --- CHANGELOG.md | 1 + .../cluster/settings/ClusterSettingsIT.java | 108 ++++++++++++++++++ .../common/settings/ClusterSettings.java | 5 +- .../main/java/org/opensearch/node/Node.java | 1 + .../org/opensearch/threadpool/ThreadPool.java | 101 ++++++++++++++++ .../threadpool/ThreadPoolTests.java | 44 +++++++ 6 files changed, 259 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 668846cc1da6a..5bf90a75d0148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) - Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) - Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) +- Add support to dynamically resize threadpools size. ([#16236](https://github.com/opensearch-project/OpenSearch/pull/16236)) - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java index 541f1048bb246..59542f886dcac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java @@ -380,6 +380,114 @@ public void testMissingUnits() { } } + public void testThreadPoolSettings() { + // wrong threadpool + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.wrong.max", "-1").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertTrue(ex.getCause().getMessage().contains("illegal thread_pool name : ")); + } + + // Scaling threadpool - negative value + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.snapshot.max", "-1").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "illegal value for [cluster.thread_pool.snapshot], has to be positive value"); + } + + // Scaling threadpool - Other than max and core + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.snapshot.wrong", "-1").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "illegal thread_pool config : [wrong] should only have [core, max]"); + } + + // Scaling threadpool - core > max + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings( + Settings.builder().put("cluster.thread_pool.snapshot.core", "2").put("cluster.thread_pool.snapshot.max", "1").build() + ) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "core threadpool size cannot be greater than max"); + } + + // Scaling threadpool - Max value lesser than default value of 4 + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.generic.max", "1").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "core threadpool size cannot be greater than max"); + } + + // Scaling threadpool - happy case - transient overrides persistent + ClusterUpdateSettingsResponse clusterUpdateSettingsResponse = client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings( + Settings.builder().put("cluster.thread_pool.snapshot.core", "2").put("cluster.thread_pool.snapshot.max", "2").build() + ) + .setPersistentSettings(Settings.builder().put("cluster.thread_pool.snapshot.max", "1").build()) + .get(); + assertTrue(clusterUpdateSettingsResponse.isAcknowledged()); + + // Fixed threadpool - Other than size + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.get.wrong", "-1").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "illegal thread_pool config : [wrong] should only have [size]"); + } + + // Fixed threadpool - 0 value + try { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.get.size", "0").build()) + .get(); + fail("bogus value"); + } catch (IllegalArgumentException ex) { + assertEquals(ex.getCause().getMessage(), "illegal value for [cluster.thread_pool.get], has to be positive value"); + } + + // Fixed threadpool - happy case + clusterUpdateSettingsResponse = client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put("cluster.thread_pool.get.size", "1").build()) + .setPersistentSettings(Settings.builder().put("cluster.thread_pool.get.size", "1").build()) + .get(); + assertTrue(clusterUpdateSettingsResponse.isAcknowledged()); + } + public void testLoggerLevelUpdate() { assertAcked(prepareCreate("test")); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index a84a29256ee19..f769f8729c25b 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -805,7 +805,10 @@ public void apply(Settings value, Settings current, Settings previous) { // Settings to be used for limiting rest requests ResponseLimitSettings.CAT_INDICES_RESPONSE_LIMIT_SETTING, ResponseLimitSettings.CAT_SHARDS_RESPONSE_LIMIT_SETTING, - ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING + ResponseLimitSettings.CAT_SEGMENTS_RESPONSE_LIMIT_SETTING, + + // Thread pool Settings + ThreadPool.CLUSTER_THREAD_POOL_SIZE_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 584d95b9ff6b5..e74fca60b0201 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -624,6 +624,7 @@ protected Node( additionalSettingsFilter, settingsUpgraders ); + threadPool.registerClusterSettingsListeners(settingsModule.getClusterSettings()); scriptModule.registerClusterSettingsListeners(scriptService, settingsModule.getClusterSettings()); final NetworkService networkService = new NetworkService( getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class)) diff --git a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java index e0b15e54f6e2e..269a4c87dfb72 100644 --- a/server/src/main/java/org/opensearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/opensearch/threadpool/ThreadPool.java @@ -38,6 +38,7 @@ import org.opensearch.Version; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.SizeValue; @@ -58,11 +59,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; @@ -122,6 +126,9 @@ public static class Names { public static final String REMOTE_STATE_CHECKSUM = "remote_state_checksum"; } + static Set scalingThreadPoolKeys = new HashSet<>(Arrays.asList("max", "core")); + static Set fixedThreadPoolKeys = new HashSet<>(Arrays.asList("size")); + /** * The threadpool type. * @@ -222,6 +229,12 @@ public Collection builders() { Setting.Property.NodeScope ); + public static final Setting CLUSTER_THREAD_POOL_SIZE_SETTING = Setting.groupSetting( + "cluster.thread_pool.", + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public ThreadPool(final Settings settings, final ExecutorBuilder... customBuilders) { this(settings, null, customBuilders); } @@ -403,6 +416,94 @@ public Info info(String name) { return holder.info; } + public void registerClusterSettingsListeners(ClusterSettings clusterSettings) { + clusterSettings.addSettingsUpdateConsumer(CLUSTER_THREAD_POOL_SIZE_SETTING, this::setThreadPool, this::validateSetting); + } + + /* + Scaling threadpool can provide only max and core + Fixed/ResizableQueue can provide only size + + For example valid settings would be for scaling and fixed thead pool + cluster.threadpool.snapshot.max : "5", + cluster.threadpool.snapshot.core : "5", + cluster.threadpool.get.size : "2", + */ + private void validateSetting(Settings tpSettings) { + Map tpGroups = tpSettings.getAsGroups(); + for (Map.Entry entry : tpGroups.entrySet()) { + String tpName = entry.getKey(); + if (THREAD_POOL_TYPES.containsKey(tpName) == false) { + throw new IllegalArgumentException("illegal thread_pool name : " + tpName); + } + Settings tpGroup = entry.getValue(); + ExecutorHolder holder = executors.get(tpName); + assert holder.executor instanceof OpenSearchThreadPoolExecutor; + OpenSearchThreadPoolExecutor threadPoolExecutor = (OpenSearchThreadPoolExecutor) holder.executor; + if (holder.info.type == ThreadPoolType.SCALING) { + if (scalingThreadPoolKeys.containsAll(tpGroup.keySet()) == false) { + throw new IllegalArgumentException( + "illegal thread_pool config : " + tpGroup.keySet() + " should only have " + scalingThreadPoolKeys + ); + } + int max = tpGroup.getAsInt("max", threadPoolExecutor.getMaximumPoolSize()); + int core = tpGroup.getAsInt("core", threadPoolExecutor.getCorePoolSize()); + if (core < 1 || max < 1) { + throw new IllegalArgumentException("illegal value for [cluster.thread_pool." + tpName + "], has to be positive value"); + } else if (core > max) { + throw new IllegalArgumentException("core threadpool size cannot be greater than max"); + } + } else { + if (fixedThreadPoolKeys.containsAll(tpGroup.keySet()) == false) { + throw new IllegalArgumentException( + "illegal thread_pool config : " + tpGroup.keySet() + " should only have " + fixedThreadPoolKeys + ); + } + int size = tpGroup.getAsInt("size", threadPoolExecutor.getMaximumPoolSize()); + if (size < 1) { + throw new IllegalArgumentException("illegal value for [cluster.thread_pool." + tpName + "], has to be positive value"); + } + } + } + } + + public void setThreadPool(Settings tpSettings) { + Map tpGroups = tpSettings.getAsGroups(); + for (Map.Entry entry : tpGroups.entrySet()) { + String tpName = entry.getKey(); + Settings tpGroup = entry.getValue(); + ExecutorHolder holder = executors.get(tpName); + assert holder.executor instanceof OpenSearchThreadPoolExecutor; + OpenSearchThreadPoolExecutor executor = (OpenSearchThreadPoolExecutor) holder.executor; + if (holder.info.type == ThreadPoolType.SCALING) { + int max = tpGroup.getAsInt("max", executor.getMaximumPoolSize()); + int core = tpGroup.getAsInt("core", executor.getCorePoolSize()); + /* + If we are decreasing, core pool size has to be decreased first. + If we are increasing ,max pool size has to be increased first + This ensures that core pool is always smaller than max pool size . + Other wise IllegalArgumentException will be thrown from ThreadPoolExecutor + */ + if (core < executor.getCorePoolSize()) { + executor.setCorePoolSize(core); + executor.setMaximumPoolSize(max); + } else { + executor.setMaximumPoolSize(max); + executor.setCorePoolSize(core); + } + } else { + int size = tpGroup.getAsInt("size", executor.getMaximumPoolSize()); + if (size < executor.getCorePoolSize()) { + executor.setCorePoolSize(size); + executor.setMaximumPoolSize(size); + } else { + executor.setMaximumPoolSize(size); + executor.setCorePoolSize(size); + } + } + } + } + public ThreadPoolStats stats() { List stats = new ArrayList<>(); for (ExecutorHolder holder : executors.values()) { diff --git a/server/src/test/java/org/opensearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/opensearch/threadpool/ThreadPoolTests.java index 658de5ec49500..205bf7621c576 100644 --- a/server/src/test/java/org/opensearch/threadpool/ThreadPoolTests.java +++ b/server/src/test/java/org/opensearch/threadpool/ThreadPoolTests.java @@ -36,6 +36,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.FutureUtils; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor; import org.opensearch.test.OpenSearchTestCase; import java.util.concurrent.CountDownLatch; @@ -152,4 +153,47 @@ public void testInheritContextOnSchedule() throws InterruptedException { terminate(threadPool); } } + + public void testThreadPoolResize() { + TestThreadPool threadPool = new TestThreadPool("test"); + try { + // increase it + Settings commonSettings = Settings.builder().put("snapshot.max", "10").put("snapshot.core", "2").put("get.size", "100").build(); + threadPool.setThreadPool(commonSettings); + ExecutorService executorService = threadPool.executor("snapshot"); + OpenSearchThreadPoolExecutor executor = (OpenSearchThreadPoolExecutor) executorService; + assertEquals(10, executor.getMaximumPoolSize()); + assertEquals(2, executor.getCorePoolSize()); + + executorService = threadPool.executor("get"); + executor = (OpenSearchThreadPoolExecutor) executorService; + assertEquals(100, executor.getMaximumPoolSize()); + assertEquals(100, executor.getCorePoolSize()); + + // decrease it + commonSettings = Settings.builder().put("snapshot.max", "2").put("snapshot.core", "1").put("get.size", "90").build(); + threadPool.setThreadPool(commonSettings); + executorService = threadPool.executor("snapshot"); + executor = (OpenSearchThreadPoolExecutor) executorService; + assertEquals(2, executor.getMaximumPoolSize()); + assertEquals(1, executor.getCorePoolSize()); + + executorService = threadPool.executor("get"); + executor = (OpenSearchThreadPoolExecutor) executorService; + assertEquals(90, executor.getMaximumPoolSize()); + assertEquals(90, executor.getCorePoolSize()); + } finally { + terminate(threadPool); + } + } + + public void testThreadPoolResizeFail() { + TestThreadPool threadPool = new TestThreadPool("test"); + try { + Settings commonSettings = Settings.builder().put("snapshot.max", "50").put("snapshot.core", "100").build(); + assertThrows(IllegalArgumentException.class, () -> threadPool.setThreadPool(commonSettings)); + } finally { + terminate(threadPool); + } + } } From 23d1c7a55a63250b962c1fad4e6fb962fdd156cc Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 15 Oct 2024 21:13:33 +0530 Subject: [PATCH 240/357] Fix deletion permits flow in RemoteFsTimestampAwareTranslog (#16282) --------- Signed-off-by: Sachin Kale --- .../RemoteFsTimestampAwareTranslog.java | 52 ++++- .../index/translog/RemoteFsTranslog.java | 9 +- .../transfer/TranslogTransferManager.java | 84 +++---- .../RemoteFsTimestampAwareTranslogTests.java | 213 +++++++++++++++++- .../index/translog/RemoteFsTranslogTests.java | 24 ++ 5 files changed, 317 insertions(+), 65 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 3ccacde22bbfc..1f54c09a04cc7 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -215,21 +215,42 @@ public void onResponse(List blobMetadata) { logger.debug(() -> "generationsToBeDeleted = " + generationsToBeDeleted); if (generationsToBeDeleted.isEmpty() == false) { // Delete stale generations - translogTransferManager.deleteGenerationAsync( - primaryTermSupplier.getAsLong(), - generationsToBeDeleted, - remoteGenerationDeletionPermits::release - ); + try { + translogTransferManager.deleteGenerationAsync( + primaryTermSupplier.getAsLong(), + generationsToBeDeleted, + remoteGenerationDeletionPermits::release + ); + } catch (Exception e) { + logger.error("Exception in delete generations flow", e); + // Release permit that is meant for metadata files and return + remoteGenerationDeletionPermits.release(); + assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits " + + remoteGenerationDeletionPermits.availablePermits() + + " is not equal to " + + REMOTE_DELETION_PERMITS; + return; + } } else { remoteGenerationDeletionPermits.release(); } if (metadataFilesToBeDeleted.isEmpty() == false) { // Delete stale metadata files - translogTransferManager.deleteMetadataFilesAsync( - metadataFilesToBeDeleted, - remoteGenerationDeletionPermits::release - ); + try { + translogTransferManager.deleteMetadataFilesAsync( + metadataFilesToBeDeleted, + remoteGenerationDeletionPermits::release + ); + } catch (Exception e) { + logger.error("Exception in delete metadata files flow", e); + // Permits is already released by deleteMetadataFilesAsync + assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits " + + remoteGenerationDeletionPermits.availablePermits() + + " is not equal to " + + REMOTE_DELETION_PERMITS; + return; + } // Update cache to keep only those metadata files that are not getting deleted oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted); @@ -240,7 +261,12 @@ public void onResponse(List blobMetadata) { remoteGenerationDeletionPermits.release(); } } catch (Exception e) { + logger.error("Exception in trimUnreferencedReaders", e); remoteGenerationDeletionPermits.release(REMOTE_DELETION_PERMITS); + assert remoteGenerationDeletionPermits.availablePermits() == REMOTE_DELETION_PERMITS : "Available permits " + + remoteGenerationDeletionPermits.availablePermits() + + " is not equal to " + + REMOTE_DELETION_PERMITS; } } @@ -441,7 +467,8 @@ protected static void deleteStaleRemotePrimaryTerms( } Optional minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> { try { - return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1(); + return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap, logger) + .v1(); } catch (IOException e) { return Long.MIN_VALUE; } @@ -482,7 +509,8 @@ protected static Long getMinPrimaryTermInRemote( protected static Tuple getMinMaxPrimaryTermFromMetadataFile( String metadataFile, TranslogTransferManager translogTransferManager, - Map> oldFormatMetadataFilePrimaryTermMap + Map> oldFormatMetadataFilePrimaryTermMap, + Logger logger ) throws IOException { Tuple minMaxPrimaryTermFromFileName = TranslogTransferMetadata.getMinMaxPrimaryTermFromFilename(metadataFile); if (minMaxPrimaryTermFromFileName != null) { @@ -504,6 +532,8 @@ protected static Tuple getMinMaxPrimaryTermFromMetadataFile( if (primaryTerm.isPresent()) { minPrimaryTem = primaryTerm.get(); } + } else { + logger.warn("No primary term found from GenerationToPrimaryTermMap for file [{}]", metadataFile); } Tuple minMaxPrimaryTermTuple = new Tuple<>(minPrimaryTem, maxPrimaryTem); oldFormatMetadataFilePrimaryTermMap.put(metadataFile, minMaxPrimaryTermTuple); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index f5a9ed8ed9362..e697e16d5e8a0 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -590,7 +590,14 @@ protected void trimUnreferencedReaders(boolean onlyTrimLocal) throws IOException generationsToDelete.add(generation); } if (generationsToDelete.isEmpty() == false) { - deleteRemoteGeneration(generationsToDelete); + try { + deleteRemoteGeneration(generationsToDelete); + } catch (Exception e) { + logger.error("Exception in delete generations flow", e); + // Release permit that is meant for metadata files and return + remoteGenerationDeletionPermits.release(); + return; + } translogTransferManager.deleteStaleTranslogMetadataFilesAsync(remoteGenerationDeletionPermits::release); deleteStaleRemotePrimaryTerms(); } else { diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index 291218ea47499..924669d0e46a9 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -496,19 +496,24 @@ public byte[] getMetadataBytes(TranslogTransferMetadata metadata) throws IOExcep * @param onCompletion runnable to run on completion of deletion regardless of success/failure. */ public void deleteGenerationAsync(long primaryTerm, Set generations, Runnable onCompletion) { - List translogFiles = new ArrayList<>(); - generations.forEach(generation -> { - // Add .ckp and .tlog file to translog file list which is located in basePath/ - String ckpFileName = Translog.getCommitCheckpointFileName(generation); - String translogFileName = Translog.getFilename(generation); - if (isTranslogMetadataEnabled == false) { - translogFiles.addAll(List.of(ckpFileName, translogFileName)); - } else { - translogFiles.add(translogFileName); - } - }); - // Delete the translog and checkpoint files asynchronously - deleteTranslogFilesAsync(primaryTerm, translogFiles, onCompletion); + try { + List translogFiles = new ArrayList<>(); + generations.forEach(generation -> { + // Add .ckp and .tlog file to translog file list which is located in basePath/ + String ckpFileName = Translog.getCommitCheckpointFileName(generation); + String translogFileName = Translog.getFilename(generation); + if (isTranslogMetadataEnabled == false) { + translogFiles.addAll(List.of(ckpFileName, translogFileName)); + } else { + translogFiles.add(translogFileName); + } + }); + // Delete the translog and checkpoint files asynchronously + deleteTranslogFilesAsync(primaryTerm, translogFiles, onCompletion); + } catch (Exception e) { + onCompletion.run(); + throw e; + } } /** @@ -658,37 +663,32 @@ public void deleteTranslogFiles() throws IOException { * @param onCompletion runnable to run on completion of deletion regardless of success/failure. */ private void deleteTranslogFilesAsync(long primaryTerm, List files, Runnable onCompletion) { - try { - transferService.deleteBlobsAsync( - ThreadPool.Names.REMOTE_PURGE, - remoteDataTransferPath.add(String.valueOf(primaryTerm)), - files, - new ActionListener<>() { - @Override - public void onResponse(Void unused) { - fileTransferTracker.delete(files); - logger.trace("Deleted translogs for primaryTerm={} files={}", primaryTerm, files); - onCompletion.run(); - } + transferService.deleteBlobsAsync( + ThreadPool.Names.REMOTE_PURGE, + remoteDataTransferPath.add(String.valueOf(primaryTerm)), + files, + new ActionListener<>() { + @Override + public void onResponse(Void unused) { + fileTransferTracker.delete(files); + logger.trace("Deleted translogs for primaryTerm={} files={}", primaryTerm, files); + onCompletion.run(); + } - @Override - public void onFailure(Exception e) { - onCompletion.run(); - logger.error( - () -> new ParameterizedMessage( - "Exception occurred while deleting translog for primaryTerm={} files={}", - primaryTerm, - files - ), - e - ); - } + @Override + public void onFailure(Exception e) { + onCompletion.run(); + logger.error( + () -> new ParameterizedMessage( + "Exception occurred while deleting translog for primaryTerm={} files={}", + primaryTerm, + files + ), + e + ); } - ); - } catch (Exception e) { - onCompletion.run(); - throw e; - } + } + ); } /** diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 73db3314f4d1e..0995f2e75a17a 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -63,6 +63,7 @@ import org.mockito.Mockito; +import static org.opensearch.index.translog.RemoteFsTranslog.REMOTE_DELETION_PERMITS; import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy; import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED; @@ -480,10 +481,7 @@ public void onResponse(List blobMetadataList) { // we will not delete them if (dataFilesAfterTrim.equals(dataFilesBeforeTrim) == false) { // We check for number of pinned timestamp or +1 due to latest metadata. - assertTrue( - metadataFilesAfterTrim.size() == pinnedTimestamps.size() - || metadataFilesAfterTrim.size() == pinnedTimestamps.size() + 1 - ); + assertTrue(metadataFilesAfterTrim.size() >= pinnedTimestamps.size()); } for (String md : pinnedTimestampMatchingMetadataFiles) { @@ -1061,15 +1059,14 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class); - RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog; - // Fetch generations directly from the filename assertEquals( new Tuple<>(1L, 1008L), RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( "metadata__9223372036854774799__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1", translogTransferManager, - new HashMap<>() + new HashMap<>(), + logger ) ); assertEquals( @@ -1077,7 +1074,8 @@ public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( "metadata__9223372036854775800__9223372036854775800__9223370311919910398__31__9223372036854775803__4__1", translogTransferManager, - new HashMap<>() + new HashMap<>(), + logger ) ); assertEquals( @@ -1085,7 +1083,8 @@ public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( "metadata__9223372036854775797__9223372036854775800__9223370311919910398__31__9223372036854775803__10__1", translogTransferManager, - new HashMap<>() + new HashMap<>(), + logger ) ); @@ -1099,7 +1098,8 @@ public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1", translogTransferManager, - new HashMap<>() + new HashMap<>(), + logger ) ); assertEquals( @@ -1107,7 +1107,8 @@ public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", translogTransferManager, - Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L)) + Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L)), + logger ) ); @@ -1115,6 +1116,36 @@ public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException { verify(translogTransferManager, times(0)).readMetadata( "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1" ); + + // Older md files with empty GenerationToPrimaryTermMap + md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of()); + when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1")) + .thenReturn(md1); + assertEquals( + new Tuple<>(-1L, 2L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1", + translogTransferManager, + new HashMap<>(), + logger + ) + ); + + // Older md files with empty GenerationToPrimaryTermMap + md1 = mock(TranslogTransferMetadata.class); + when(md1.getGenerationToPrimaryTermMapper()).thenReturn(null); + when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1")) + .thenReturn(md1); + assertEquals( + new Tuple<>(-1L, 2L), + RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile( + "metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1", + translogTransferManager, + new HashMap<>(), + logger + ) + ); } public void testDeleteStaleRemotePrimaryTerms() throws IOException { @@ -1332,4 +1363,164 @@ public void testGetMinPrimaryTermInRemoteNotFetched() throws IOException { ); verify(translogTransferManager).listPrimaryTermsInRemote(); } + + public void testTrimUnreferencedReadersStalePinnedTimestamps() throws Exception { + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + + // First reader is created at the init of translog + assertEquals(3, translog.readers.size()); + assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertBusy(() -> { + assertEquals(6, translog.allUploaded().size()); + assertEquals( + 6, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> { + assertEquals( + 10, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + translog.setMinSeqNoToKeep(3); + translog.trimUnreferencedReaders(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + translog.setMinSeqNoToKeep(6); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + assertEquals(1, translog.readers.size()); + assertBusy(() -> { + assertEquals(2, translog.allUploaded().size()); + assertEquals(7, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 16, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + translog.trimUnreferencedReaders(); + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + + assertEquals(3, translog.readers.size()); + assertBusy(() -> { + assertEquals(6, translog.allUploaded().size()); + assertEquals(9, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 20, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + } + + public void testTrimUnreferencedReadersNoPermits() throws Exception { + // Acquire the permits so that remote translog deletion will not happen + translog.remoteGenerationDeletionPermits.acquire(REMOTE_DELETION_PERMITS); + + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + + // First reader is created at the init of translog + assertEquals(3, translog.readers.size()); + assertEquals(2, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertBusy(() -> { + assertEquals(6, translog.allUploaded().size()); + assertEquals( + 6, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + + assertBusy(() -> { + assertEquals( + 10, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }); + + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(3); + translog.trimUnreferencedReaders(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("4", 4, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("5", 5, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("6", 6, primaryTerm.get(), new byte[] { 1 })); + + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.setMinSeqNoToKeep(6); + translog.trimUnreferencedReaders(); + + assertEquals(1, translog.readers.size()); + assertBusy(() -> { + assertEquals(2, translog.allUploaded().size()); + assertEquals(7, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 16, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("7", 7, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("8", 8, primaryTerm.get(), new byte[] { 1 })); + + // Fetch pinned timestamps so that it won't be stale + updatePinnedTimstampTask.run(); + translog.trimUnreferencedReaders(); + + assertEquals(3, translog.readers.size()); + assertBusy(() -> { + assertEquals(6, translog.allUploaded().size()); + assertEquals(9, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals( + 20, + blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() + ); + }, 30, TimeUnit.SECONDS); + } + + public void testTrimUnreferencedReadersFailAlwaysRepo() throws Exception { + ArrayList ops = new ArrayList<>(); + + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("0", 0, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 1, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 2, primaryTerm.get(), new byte[] { 1 })); + addToTranslogAndListAndUpload(translog, ops, new Translog.Index("3", 3, primaryTerm.get(), new byte[] { 1 })); + + translog.setMinSeqNoToKeep(2); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.ZERO); + updatePinnedTimstampTask.run(); + + fail.failAlways(); + translog.trimUnreferencedReaders(); + + assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); + } } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 03c77a9a83f57..190af714d5764 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -20,6 +20,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.fs.FsBlobContainer; @@ -32,6 +33,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; @@ -1965,6 +1967,28 @@ public void writeBlobAtomic(final String blobName, final InputStream inputStream } super.writeBlobAtomic(blobName, inputStream, blobSize, failIfAlreadyExists); } + + @Override + public void listBlobsByPrefixInSortedOrder( + String blobNamePrefix, + int limit, + BlobNameSortOrder blobNameSortOrder, + ActionListener> listener + ) { + if (fail.fail()) { + listener.onFailure(new RuntimeException("blob container throwing error")); + return; + } + if (slowDown.getSleepSeconds() > 0) { + try { + Thread.sleep(slowDown.getSleepSeconds() * 1000L); + } catch (InterruptedException e) { + listener.onFailure(new RuntimeException(e)); + return; + } + } + super.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder, listener); + } } class TranslogThread extends Thread { From a853b758135ddbf48e9bc5cf50cf75db7d197c41 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 16 Oct 2024 10:44:49 +0800 Subject: [PATCH 241/357] Fix wrong default value when setting index.number_of_routing_shards to null on index creation (#16331) * Fix wrong value when setting index.number_of_routing_shards to null on index creation Signed-off-by: Gao Binlong * Modify change log Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../test/indices.create/10_basic.yml | 30 +++++++++++++++++++ .../metadata/MetadataCreateIndexService.java | 2 +- .../MetadataCreateIndexServiceTests.java | 18 +++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf90a75d0148..73db9bb9ed7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) +- Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index 0f8c7a7a68f07..077ea98ccf8cc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -112,3 +112,33 @@ properties: "": type: keyword + +--- +"Create index with setting index.number_of_routing_shards to null": + - skip: + version: " - 2.99.99" + reason: "fixed in 3.0.0" + - do: + indices.create: + index: test_index + body: + settings: + number_of_routing_shards: null + - do: + cluster.state: + metric: [ metadata ] + index: test_index + - match : { metadata.indices.test_index.routing_num_shards: 1024 } + + - do: + indices.create: + index: test_index1 + body: + settings: + number_of_routing_shards: null + number_of_shards: 3 + - do: + cluster.state: + metric: [ metadata ] + index: test_index1 + - match : { metadata.indices.test_index1.routing_num_shards: 768 } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 8b08927bc146a..abda5dad25e4e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1210,7 +1210,7 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index // in this case we either have no index to recover from or // we have a source index with 1 shard and without an explicit split factor // or one that is valid in that case we can split into whatever and auto-generate a new factor. - if (IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettings)) { + if (indexSettings.get(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()) != null) { routingNumShards = IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(indexSettings); } else { routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 3f223706819b7..1fdd038053eb6 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1857,6 +1857,24 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } + public void testGetIndexNumberOfRoutingShardsWhenExplicitlySetToNull() { + String nullValue = null; + Settings indexSettings = Settings.builder() + .put("index.version.created", Version.CURRENT) + .put(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), nullValue) + .build(); + int targetRoutingNumberOfShards = getIndexNumberOfRoutingShards(indexSettings, null); + assertThat(targetRoutingNumberOfShards, is(1024)); + + indexSettings = Settings.builder() + .put("index.version.created", Version.CURRENT) + .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3) + .put(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), nullValue) + .build(); + targetRoutingNumberOfShards = getIndexNumberOfRoutingShards(indexSettings, null); + assertThat(targetRoutingNumberOfShards, is(768)); + } + public void testSoftDeletesDisabledIsRejected() { final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); From d404359fbce251e654f4d59b318bf3ae8228441e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 16 Oct 2024 00:24:18 -0400 Subject: [PATCH 242/357] JDK-23: Mitigation for https://bugs.openjdk.org/browse/JDK-8341127 issue (performance regression) (#16342) Signed-off-by: Andriy Redko --- distribution/src/config/jvm.options | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index f0ac98faffda9..a8c96f33ce51d 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -85,3 +85,7 @@ ${error.file} # HDFS ForkJoinPool.common() support by SecurityManager -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory + +# See please https://bugs.openjdk.org/browse/JDK-8341127 (openjdk/jdk#21283) +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache +23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached From dd5a87aa2bf734767f7e726a01644526a0f491b6 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 16 Oct 2024 18:47:03 +0530 Subject: [PATCH 243/357] Bugfix in snapshot V2 restore flow (#16332) Signed-off-by: Sachin Kale --- ...rePinnedTimestampsGarbageCollectionIT.java | 6 +- .../RestoreShallowSnapshotV2IT.java | 91 +++++++++++++++++++ .../store/RemoteSegmentStoreDirectory.java | 13 ++- .../RemoteFsTimestampAwareTranslog.java | 2 +- .../RemoteFsTimestampAwareTranslogTests.java | 4 +- 5 files changed, 109 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java index 08ece7df457cc..2922ad33586d2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsGarbageCollectionIT.java @@ -421,8 +421,8 @@ public void testIndexDeletionNoPinnedTimestamps() throws Exception { client().admin().indices().prepareDelete(INDEX_NAME).get(); assertBusy(() -> { - assertEquals(0, Files.list(translogMetadataPath).collect(Collectors.toList()).size()); - assertEquals(0, Files.list(translogDataPath).collect(Collectors.toList()).size()); + assertEquals(1, Files.list(translogMetadataPath).collect(Collectors.toList()).size()); + assertEquals(4, Files.list(translogDataPath).collect(Collectors.toList()).size()); }); } @@ -490,7 +490,7 @@ public void testIndexDeletionWithPinnedTimestamps() throws Exception { assertBusy(() -> { List metadataFiles = Files.list(translogMetadataPath).collect(Collectors.toList()); - assertEquals(1, metadataFiles.size()); + assertEquals(2, metadataFiles.size()); verifyTranslogDataFileCount(metadataFiles, translogDataPath); }); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index 24f1141ddbede..d532abaa2b0ad 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -27,22 +27,26 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.index.Index; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.remote.RemoteStoreEnums; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; import org.opensearch.snapshots.SnapshotInfo; import org.opensearch.snapshots.SnapshotRestoreException; import org.opensearch.snapshots.SnapshotState; +import org.opensearch.test.BackgroundIndexer; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; @@ -53,15 +57,18 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; @@ -885,4 +892,88 @@ public void testRestoreOperationsUsingDifferentRepos() throws Exception { ensureGreen(indexName1); assertDocsPresentInIndex(client, indexName1, 3 * numDocsInIndex1); } + + public void testContinuousIndexing() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNode(); + String index = "test-index"; + String snapshotRepo = "test-restore-snapshot-repo"; + String baseSnapshotName = "snapshot_"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true)); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + + createIndex(index, indexSettings); + ensureGreen(index); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(index) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(randomIntBetween(1, 5))); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueSeconds(randomIntBetween(1, 5))); + + long totalDocs = 0; + Map snapshots = new HashMap<>(); + int numDocs = randomIntBetween(200, 300); + totalDocs += numDocs; + try (BackgroundIndexer indexer = new BackgroundIndexer(index, MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) { + int numberOfSnapshots = 5; + for (int i = 0; i < numberOfSnapshots; i++) { + logger.info("--> waiting for {} docs to be indexed ...", numDocs); + long finalTotalDocs1 = totalDocs; + assertBusy(() -> assertEquals(finalTotalDocs1, indexer.totalIndexedDocs()), 120, TimeUnit.SECONDS); + logger.info("--> {} total docs indexed", totalDocs); + String snapshotName = baseSnapshotName + i; + createSnapshot(snapshotRepo, snapshotName, new ArrayList<>()); + snapshots.put(snapshotName, totalDocs); + if (i < numberOfSnapshots - 1) { + numDocs = randomIntBetween(200, 300); + indexer.continueIndexing(numDocs); + totalDocs += numDocs; + } + } + } + + logger.info("Snapshots Status: " + snapshots); + + for (String snapshot : snapshots.keySet()) { + logger.info("Restoring snapshot: {}", snapshot); + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(index)).get()); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshot) + .setWaitForCompletion(true) + .setIndices() + .get(); + + assertEquals(RestStatus.OK, restoreSnapshotResponse1.status()); + + // Verify restored index's stats + ensureGreen(TimeValue.timeValueSeconds(60), index); + long finalTotalDocs = totalDocs; + assertBusy(() -> { + Long hits = client().prepareSearch(index) + .setQuery(matchAllQuery()) + .setSize((int) finalTotalDocs) + .storedFields() + .execute() + .actionGet() + .getHits() + .getTotalHits().value; + + assertEquals(snapshots.get(snapshot), hits); + }); + } + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 25d6e24cade14..27a78dc3ce2f6 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -39,6 +39,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.threadpool.ThreadPool; @@ -891,6 +892,16 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException lastSuccessfulFetchOfPinnedTimestamps ); + if (metadataFilesEligibleToDelete.isEmpty()) { + logger.debug("No metadata files are eligible to be deleted based on lastNMetadataFilesToKeep and age"); + return; + } + + // If pinned timestamps are enabled, make sure to not delete last metadata file. + if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { + metadataFilesEligibleToDelete.remove(sortedMetadataFileList.get(0)); + } + List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream() .filter(metadataFile -> allLockFiles.contains(metadataFile) == false) .collect(Collectors.toList()); @@ -905,7 +916,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException Set activeSegmentRemoteFilenames = new HashSet<>(); final Set metadataFilesToFilterActiveSegments = getMetadataFilesToFilterActiveSegments( - lastNMetadataFilesToKeep, + sortedMetadataFileList.indexOf(metadataFilesEligibleToDelete.get(0)), sortedMetadataFileList, allLockFiles ); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 1f54c09a04cc7..54cbf8ac9a9f8 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -189,7 +189,7 @@ public void onResponse(List blobMetadata) { List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted); // If index is not deleted, make sure to keep latest metadata file - if (indexDeleted == false) { + if (indexDeleted == false || RemoteStoreSettings.isPinnedTimestampsEnabled()) { metadataFilesToBeDeleted.remove(metadataFiles.get(0)); } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 0995f2e75a17a..838f97ade9e8e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -286,9 +286,9 @@ public void testIndexDeletionWithNoPinnedTimestampNoRecentMdFiles() throws Excep assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertBusy(() -> { - assertEquals(0, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); assertEquals( - 0, + 12, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); }); From 62081f2e2686b9d9bf3ac71249bf0f5a7b415401 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Wed, 16 Oct 2024 21:51:05 +0800 Subject: [PATCH 244/357] fix cluster not able to spin up issue when disk usage exceeds threshold (#15258) * fix cluster not able to spin up issue when disk usage exceeds threshold Signed-off-by: zane-neo * Add comment to changes Signed-off-by: zane-neo * Add UT to ensure the keepAliveThread starts before node starts Signed-off-by: zane-neo * remove unused imports Signed-off-by: zane-neo * Fix forbidden API calls check failed issue Signed-off-by: zane-neo * format code Signed-off-by: zane-neo * format code Signed-off-by: zane-neo * change setInstance method to static Signed-off-by: zane-neo * Add countdownlatch in test to coordinate the thread to avoid concureency issue caused test failure Signed-off-by: zane-neo --------- Signed-off-by: zane-neo --- CHANGELOG.md | 2 + .../opensearch/bootstrap/BootstrapTests.java | 42 +++++++++++++++++++ .../org/opensearch/bootstrap/Bootstrap.java | 21 +++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73db9bb9ed7af..c581f9ae8811e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) +- Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) + ### Security diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index e9219de218aef..7aa63a2736a8c 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,6 +31,7 @@ package org.opensearch.bootstrap; +import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -38,6 +39,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; +import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -51,8 +53,14 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class BootstrapTests extends OpenSearchTestCase { Environment env; @@ -131,4 +139,38 @@ private void assertPassphraseRead(String source, String expected) { } } + public void testInitExecutionOrder() throws Exception { + AtomicInteger order = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(1); + Thread mockThread = new Thread(() -> { + assertEquals(0, order.getAndIncrement()); + countDownLatch.countDown(); + }); + + Node mockNode = mock(Node.class); + doAnswer(invocation -> { + try { + boolean threadStarted = countDownLatch.await(1000, TimeUnit.MILLISECONDS); + assertTrue( + "Waited for one second but the keepAliveThread isn't started, please check the execution order of" + + "keepAliveThread.start and node.start", + threadStarted + ); + } catch (InterruptedException e) { + fail("Thread interrupted"); + } + assertEquals(1, order.getAndIncrement()); + return null; + }).when(mockNode).start(); + + LogConfigurator.registerErrorListener(); + Bootstrap testBootstrap = new Bootstrap(mockThread, mockNode); + Bootstrap.setInstance(testBootstrap); + + Bootstrap.startInstance(testBootstrap); + + verify(mockNode).start(); + assertEquals(2, order.get()); + } + } diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 4e167d10b99fa..757e2c9da6e49 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -93,6 +93,17 @@ final class Bootstrap { private final Thread keepAliveThread; private final Spawner spawner = new Spawner(); + // For testing purpose + static void setInstance(Bootstrap bootstrap) { + INSTANCE = bootstrap; + } + + // For testing purpose + Bootstrap(Thread keepAliveThread, Node node) { + this.keepAliveThread = keepAliveThread; + this.node = node; + } + /** creates a new instance */ Bootstrap() { keepAliveThread = new Thread(new Runnable() { @@ -336,8 +347,10 @@ private static Environment createEnvironment( } private void start() throws NodeValidationException { - node.start(); + // keepAliveThread should start first than node to ensure the cluster can spin up successfully in edge cases: + // https://github.com/opensearch-project/OpenSearch/issues/14791 keepAliveThread.start(); + node.start(); } static void stop() throws IOException { @@ -410,7 +423,7 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui throw new BootstrapException(e); } - INSTANCE.start(); + startInstance(INSTANCE); // We don't close stderr if `--quiet` is passed, because that // hides fatal startup errors. For example, if OpenSearch is @@ -462,6 +475,10 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui } } + static void startInstance(Bootstrap instance) throws NodeValidationException { + instance.start(); + } + @SuppressForbidden(reason = "System#out") private static void closeSystOut() { System.out.close(); From 659451685339d41a50c2065a543b6ca1f8e62499 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Wed, 16 Oct 2024 20:07:12 +0530 Subject: [PATCH 245/357] [Snapshot V2] Use metadata from source snapshot while cloning snapshot V2 (#16344) --------- Signed-off-by: Gaurav Bafna --- .../snapshots/CloneSnapshotV2IT.java | 113 ++++++++++++++++++ .../snapshots/SnapshotsService.java | 43 +++++-- 2 files changed, 144 insertions(+), 12 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java index 69e85b13548e0..b00a8c8d64ed9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -174,6 +175,118 @@ public void testCloneShallowCopyV2() throws Exception { assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); } + public void testCloneShallowCopyV2DeletedIndex() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(snapshotV2Settings(remoteStoreRepoPath)); + + String indexName1 = "testindex1"; + String indexName2 = "testindex2"; + String indexName3 = "testindex3"; + String snapshotRepoName = "test-clone-snapshot-repo"; + String snapshotName1 = "test-create-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + Client client = client(); + + assertAcked( + client.admin() + .cluster() + .preparePutRepository(snapshotRepoName) + .setType(FsRepository.TYPE) + .setSettings( + Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true) + .put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true) + ) + ); + + createIndex(indexName1, getRemoteStoreBackedIndexSettings()); + createIndex(indexName2, getRemoteStoreBackedIndexSettings()); + + final int numDocsInIndex1 = 10; + final int numDocsInIndex2 = 20; + indexRandomDocs(indexName1, numDocsInIndex1); + indexRandomDocs(indexName2, numDocsInIndex2); + ensureGreen(indexName1, indexName2); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(true) + .get(); + SnapshotInfo sourceSnapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(sourceSnapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(sourceSnapshotInfo.successfulShards(), greaterThan(0)); + assertThat(sourceSnapshotInfo.successfulShards(), equalTo(sourceSnapshotInfo.totalShards())); + assertThat(sourceSnapshotInfo.snapshotId().getName(), equalTo(snapshotName1)); + + // Validate that the snapshot was created + final BlobStoreRepository repository = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance( + RepositoriesService.class + ).repository(snapshotRepoName); + PlainActionFuture repositoryDataPlainActionFuture = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFuture); + + RepositoryData repositoryData = repositoryDataPlainActionFuture.get(); + + assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfo.snapshotId())); + + createIndex(indexName3, getRemoteStoreBackedIndexSettings()); + indexRandomDocs(indexName3, 10); + ensureGreen(indexName3); + + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName2)).get()); + + AcknowledgedResponse response = client().admin() + .cluster() + .prepareCloneSnapshot(snapshotRepoName, snapshotName1, "test_clone_snapshot1") + .setIndices("*") + .get(); + assertTrue(response.isAcknowledged()); + awaitClusterManagerFinishRepoOperations(); + + AtomicReference cloneSnapshotId = new AtomicReference<>(); + // Validate that snapshot is present in repository data + waitUntil(() -> { + PlainActionFuture repositoryDataPlainActionFutureClone = new PlainActionFuture<>(); + repository.getRepositoryData(repositoryDataPlainActionFutureClone); + + RepositoryData repositoryData1; + try { + repositoryData1 = repositoryDataPlainActionFutureClone.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (SnapshotId snapshotId : repositoryData1.getSnapshotIds()) { + if (snapshotId.getName().equals("test_clone_snapshot1")) { + cloneSnapshotId.set(snapshotId); + return true; + } + } + return false; + }, 90, TimeUnit.SECONDS); + + final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId.get(); + SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get( + f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal))) + ); + + assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp())); + for (String index : sourceSnapshotInfo.indices()) { + assertTrue(cloneSnapshotInfo.indices().contains(index)); + + } + assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards())); + } + public void testCloneShallowCopyAfterDisablingV2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 6688c7dd0431a..6e66f8c958666 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -885,7 +885,6 @@ public void cloneSnapshotV2( ) { long startTime = System.currentTimeMillis(); - ClusterState currentState = clusterService.state(); String snapshotName = snapshot.getSnapshotId().getName(); repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(Priority.URGENT) { private SnapshotsInProgress.Entry newEntry; @@ -963,8 +962,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); executor.execute(ActionRunnable.supply(snapshotInfoListener, () -> repository.getSnapshotInfo(sourceSnapshotId))); - final ShardGenerations shardGenerations = repositoryData.shardGenerations(); - snapshotInfoListener.whenComplete(snapshotInfo -> { final SnapshotInfo cloneSnapshotInfo = new SnapshotInfo( snapshot.getSnapshotId(), @@ -984,17 +981,28 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl throw new SnapshotException(repositoryName, snapshotName, "Aborting snapshot-v2 clone, no longer cluster manager"); } final StepListener pinnedTimestampListener = new StepListener<>(); - pinnedTimestampListener.whenComplete(repoData -> { + final StepListener metadataListener = new StepListener<>(); + pinnedTimestampListener.whenComplete( + rData -> threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(metadataListener, () -> { + final Metadata.Builder metaBuilder = Metadata.builder(repository.getSnapshotGlobalMetadata(newEntry.source())); + for (IndexId index : newEntry.indices()) { + metaBuilder.put(repository.getSnapshotIndexMetaData(repositoryData, newEntry.source(), index), false); + } + return metaBuilder.build(); + })), + e -> { + logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} ", repositoryName, snapshotName); + stateWithoutSnapshotV2(newState); + leaveRepoLoop(repositoryName); + listener.onFailure(e); + } + ); + metadataListener.whenComplete(meta -> { + ShardGenerations shardGenerations = buildGenerationsV2(newEntry, meta); repository.finalizeSnapshot( shardGenerations, repositoryData.getGenId(), - metadataForSnapshot( - currentState.metadata(), - newEntry.includeGlobalState(), - false, - newEntry.dataStreams(), - newEntry.indices() - ), + metadataForSnapshot(meta, newEntry.includeGlobalState(), false, newEntry.dataStreams(), newEntry.indices()), cloneSnapshotInfo, repositoryData.getVersion(sourceSnapshotId), state -> stateWithoutSnapshot(state, snapshot), @@ -1038,7 +1046,7 @@ public void onFailure(Exception e) { } ); }, e -> { - logger.error("Failed to update pinned timestamp for snapshot-v2 {} {} ", repositoryName, snapshotName); + logger.error("Failed to retrieve metadata for snapshot-v2 {} {} ", repositoryName, snapshotName); stateWithoutSnapshotV2(newState); leaveRepoLoop(repositoryName); listener.onFailure(e); @@ -1544,6 +1552,17 @@ private static ShardGenerations buildGenerations(SnapshotsInProgress.Entry snaps return builder.build(); } + private static ShardGenerations buildGenerationsV2(SnapshotsInProgress.Entry snapshot, Metadata metadata) { + ShardGenerations.Builder builder = ShardGenerations.builder(); + snapshot.indices().forEach(indexId -> { + int shardCount = metadata.index(indexId.getName()).getNumberOfShards(); + for (int i = 0; i < shardCount; i++) { + builder.put(indexId, i, null); + } + }); + return builder.build(); + } + private static Metadata metadataForSnapshot( Metadata metadata, boolean includeGlobalState, From ec7b6528ccfce6c638134a52f670a41a7e09fef1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 16 Oct 2024 10:42:39 -0400 Subject: [PATCH 246/357] [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures (#16337) * [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../reactor-netty-core-1.1.22.jar.sha1 | 1 - .../reactor-netty-core-1.1.23.jar.sha1 | 1 + .../reactor-netty-http-1.1.22.jar.sha1 | 1 - .../reactor-netty-http-1.1.23.jar.sha1 | 1 + .../reactor-netty-core-1.1.22.jar.sha1 | 1 - .../reactor-netty-core-1.1.23.jar.sha1 | 1 + .../reactor-netty-http-1.1.22.jar.sha1 | 1 - .../reactor-netty-http-1.1.23.jar.sha1 | 1 + .../rest/ReactorNetty4StreamingIT.java | 29 +++++++++++++++++-- .../ReactorNetty4HttpServerTransport.java | 1 + 12 files changed, 34 insertions(+), 7 deletions(-) delete mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.22.jar.sha1 create mode 100644 plugins/repository-azure/licenses/reactor-netty-core-1.1.23.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.22.jar.sha1 create mode 100644 plugins/repository-azure/licenses/reactor-netty-http-1.1.23.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.22.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.23.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.22.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.23.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index c581f9ae8811e..b328748a63ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) +- [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) - Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 4bb6f3af7fbe4..5740c124910b9 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -34,7 +34,7 @@ netty = 4.1.114.Final joda = 2.12.7 # project reactor -reactor_netty = 1.1.22 +reactor_netty = 1.1.23 reactor = 3.5.20 # client dependencies diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.22.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.22.jar.sha1 deleted file mode 100644 index cc894568c5760..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.22.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08356b59b29f86e7142c9daca0434653a64ae64b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.23.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.23.jar.sha1 new file mode 100644 index 0000000000000..8f56bb5165fa3 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.23.jar.sha1 @@ -0,0 +1 @@ +a7059b0c18ab7aa0fa9e08b48cb6a20b15c11478 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.22.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.22.jar.sha1 deleted file mode 100644 index 2402813f831ce..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.22.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2faf64b3822b0512f15d72a325e2826eb8564413 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.23.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.23.jar.sha1 new file mode 100644 index 0000000000000..5bb3136f99e93 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.23.jar.sha1 @@ -0,0 +1 @@ +94b294fa90aee2e88ad4337251e278aaac21362c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.22.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.22.jar.sha1 deleted file mode 100644 index cc894568c5760..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.22.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08356b59b29f86e7142c9daca0434653a64ae64b \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.23.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.23.jar.sha1 new file mode 100644 index 0000000000000..8f56bb5165fa3 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.23.jar.sha1 @@ -0,0 +1 @@ +a7059b0c18ab7aa0fa9e08b48cb6a20b15c11478 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.22.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.22.jar.sha1 deleted file mode 100644 index 2402813f831ce..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.22.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2faf64b3822b0512f15d72a325e2826eb8564413 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.23.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.23.jar.sha1 new file mode 100644 index 0000000000000..5bb3136f99e93 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.23.jar.sha1 @@ -0,0 +1 @@ +94b294fa90aee2e88ad4337251e278aaac21362c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java index 1b60023da0329..e022dfedb788d 100644 --- a/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java +++ b/plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingIT.java @@ -304,11 +304,36 @@ public void testStreamingLargeDocument() throws IOException { String.format( Locale.getDefault(), "{ \"index\": { \"_index\": \"test-streaming\", \"_id\": \"1\" } }\n{ \"name\": \"%s\" }\n", - randomAlphaOfLength(5000) + randomAlphaOfLength(7000) + ) + ); + + final StreamingRequest streamingRequest = new StreamingRequest<>( + "POST", + "/_bulk/stream", + Flux.fromStream(stream).map(s -> ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8))) + ); + + final StreamingResponse streamingResponse = client().streamRequest(streamingRequest); + + StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8))) + .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\"")) + .expectComplete() + .verify(); + + assertThat(streamingResponse.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(streamingResponse.getWarnings(), empty()); + } + + public void testStreamingLargeDocumentThatExceedsChunkSize() throws IOException { + final Stream stream = Stream.of( + String.format( + Locale.getDefault(), + "{ \"index\": { \"_index\": \"test-streaming\", \"_id\": \"1\" } }\n{ \"name\": \"%s\" }\n", + randomAlphaOfLength(9000) /* the default chunk size limit is set 8k */ ) ); - final Duration delay = Duration.ofMillis(1); final StreamingRequest streamingRequest = new StreamingRequest<>( "POST", "/_bulk/stream", diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java index 7f4a8f6cdef02..3dcee4e8ec045 100644 --- a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java +++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java @@ -230,6 +230,7 @@ protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Excepti spec -> spec.maxChunkSize(maxChunkSize.bytesAsInt()) .maxHeaderSize(maxHeaderSize.bytesAsInt()) .maxInitialLineLength(maxInitialLineLength.bytesAsInt()) + .allowPartialChunks(false) ) .handle((req, res) -> incomingRequest(req, res)) ); From 1a7018abf1884a287d05681358efa600471c810b Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Thu, 17 Oct 2024 04:57:36 +0300 Subject: [PATCH 247/357] Fixed inefficient Stream API call chains ending with count() (#15386) * Fixed inefficient Stream API call chains ending with count() Signed-off-by: Dmitry Kryukov * Refactored method minTermLength() as per @sandeshkr419's advice Signed-off-by: Dmitry Kryukov * Added a line in CHANGELOG.md Signed-off-by: Dmitry Kryukov --------- Signed-off-by: Dmitry Kryukov --- CHANGELOG.md | 1 + .../opensearch/percolator/QueryAnalyzer.java | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b328748a63ef3..56b1e7b4ffba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) - Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) +- Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) ### Security diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java index 3a1b6734dd444..96c022f0145a8 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/QueryAnalyzer.java @@ -510,19 +510,25 @@ static Result selectBestResult(Result result1, Result result2) { } private static int minTermLength(Set extractions) { - // In case there are only range extractions, then we return Integer.MIN_VALUE, - // so that selectBestExtraction(...) we are likely to prefer the extractions that contains at least a single extraction - if (extractions.stream().filter(queryExtraction -> queryExtraction.term != null).count() == 0 - && extractions.stream().filter(queryExtraction -> queryExtraction.range != null).count() > 0) { - return Integer.MIN_VALUE; - } - + boolean hasTerm = false; + boolean hasRange = false; int min = Integer.MAX_VALUE; + for (QueryExtraction qt : extractions) { if (qt.term != null) { + hasTerm = true; min = Math.min(min, qt.bytes().length); } + if (qt.range != null) { + hasRange = true; + } } + + // If there are no terms but there are ranges, return Integer.MIN_VALUE + if (!hasTerm && hasRange) { + return Integer.MIN_VALUE; + } + return min; } From 74dba3f54c8ead27c59a72a9becd5df811576f96 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 16 Oct 2024 23:37:38 -0400 Subject: [PATCH 248/357] Update 10_basic.yml (#16349) Signed-off-by: Andriy Redko --- .../resources/rest-api-spec/test/indices.create/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index 077ea98ccf8cc..cb222572b59fd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -116,8 +116,8 @@ --- "Create index with setting index.number_of_routing_shards to null": - skip: - version: " - 2.99.99" - reason: "fixed in 3.0.0" + version: " - 2.17.99" + reason: "fixed in 2.18.0" - do: indices.create: index: test_index From 0c3e3c009ae85943460bcc26805c1cf57b195b91 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 17 Oct 2024 13:37:10 +0530 Subject: [PATCH 249/357] Make Remote Publication a dynamic setting (#15937) * Make Remote Publication a dynamic setting Signed-off-by: Shivansh Arora Co-authored-by: Sooraj Sinha --- .../remote/RemoteStatePublicationIT.java | 279 +++++++++++++++++- .../coordination/CoordinationState.java | 10 - .../cluster/coordination/Coordinator.java | 10 +- .../InternalRemoteRoutingTableService.java | 6 +- .../RemoteRoutingTableServiceFactory.java | 4 +- .../remote/RemoteClusterStateService.java | 49 +-- .../remote/RemotePersistenceStats.java | 4 +- .../remotestore/RemoteStoreNodeAttribute.java | 2 +- .../coordination/CoordinationStateTests.java | 11 - .../RemoteClusterStateServiceTests.java | 10 +- 10 files changed, 330 insertions(+), 55 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java index 2409b5d0d0e45..0ceba16ddeecc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java @@ -8,7 +8,10 @@ package org.opensearch.gateway.remote; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; @@ -19,6 +22,8 @@ import org.opensearch.cluster.coordination.PublishClusterStateStats; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.discovery.DiscoveryStats; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; @@ -27,19 +32,32 @@ import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.recovery.RecoverySettings; +import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginsService; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.tasks.Task; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.transport.TransportChannel; +import org.opensearch.transport.TransportInterceptor; +import org.opensearch.transport.TransportRequest; +import org.opensearch.transport.TransportRequestHandler; import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -50,6 +68,8 @@ import static org.opensearch.action.admin.cluster.node.info.NodesInfoRequest.Metric.SETTINGS; import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.DISCOVERY; +import static org.opensearch.cluster.coordination.PublicationTransportHandler.PUBLISH_REMOTE_STATE_ACTION_NAME; +import static org.opensearch.cluster.coordination.PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME; import static org.opensearch.cluster.metadata.Metadata.isGlobalStateEquals; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; @@ -57,6 +77,8 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteDownloadStats.CHECKSUM_VALIDATION_FAILED_COUNT; +import static org.opensearch.gateway.remote.RemotePersistenceStats.DIFF_DOWNLOAD_STATS; +import static org.opensearch.gateway.remote.RemotePersistenceStats.FULL_DOWNLOAD_STATS; import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; @@ -88,6 +110,7 @@ public void setup() { isRemotePublicationEnabled = true; hasRemoteStateCharPrefix = randomBoolean(); hasRemoteRoutingCharPrefix = randomBoolean(); + clearInterceptedActions(); } @Override @@ -132,6 +155,13 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(InterceptingTransportService.TestPlugin.class); + return plugins; + } + public void testPublication() throws Exception { // create cluster with multi node (3 master + 2 data) prepareCluster(3, 2, INDEX_NAME, 1, 2); @@ -236,7 +266,7 @@ public void testRemotePublicationDownloadStats() { .addMetric(DISCOVERY.metricName()) .get(); - assertDataNodeDownloadStats(nodesStatsResponseDataNode); + assertDataNodeDownloadStats(nodesStatsResponseDataNode.getNodes().get(0)); } public void testRemotePublicationDisabledByRollingRestart() throws Exception { @@ -399,9 +429,162 @@ public void testVotingConfigAreCommitted() throws ExecutionException, Interrupte }); } - private void assertDataNodeDownloadStats(NodesStatsResponse nodesStatsResponse) { + public void testRemotePublicationDisabledAfterSettingChange() throws ExecutionException, InterruptedException { + prepareCluster(3, 2, Settings.EMPTY); + ensureStableCluster(5); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME); + + assertEquals(0, (int) getRequestCount(PUBLISH_STATE_ACTION_NAME)); + assertTrue(getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME) > 0); + + // disable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, false).build()) + .get(); + + clearInterceptedActions(); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + + createIndex(INDEX_NAME + "2", remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME + "2"); + + assertTrue(getRequestCount(PUBLISH_STATE_ACTION_NAME) > 0); + assertEquals(0, (int) getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME)); + } + + public void testRemotePublicationEnabledAfterSettingChange() { + isRemotePublicationEnabled = false; + prepareCluster(3, 2, Settings.EMPTY); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME); + + assertTrue(getRequestCount(PUBLISH_STATE_ACTION_NAME) > 0); + assertEquals(0, (int) getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME)); + + // enable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, true).build()) + .get(); + + clearInterceptedActions(); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + + createIndex(INDEX_NAME + "2", remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME + "2"); + + assertEquals(0, (int) getRequestCount(PUBLISH_STATE_ACTION_NAME)); + assertTrue(getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME) > 0); + } + + public void testRemotePublicationSettingChangePersistedAfterRestart() throws Exception { + isRemotePublicationEnabled = false; + prepareCluster(3, 2, Settings.EMPTY); + ensureStableCluster(5); + // enable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, true).build()) + .get(); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME); + + assertEquals(0, (int) getRequestCount(PUBLISH_STATE_ACTION_NAME)); + assertTrue(getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME) > 0); + + // disable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, false).build()) + .get(); + + clearInterceptedActions(); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + + internalCluster().rollingRestart(new InternalTestCluster.RestartCallback() { + @Override + public void doAfterNodes(int n, Client client) { + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + } + }); + + ensureStableCluster(5); + assertEquals(0, (int) getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME)); + assertTrue(getRequestCount(PUBLISH_STATE_ACTION_NAME) > 0); + } + + public void testRemotePublicationSettingChangePersistedAfterFullRestart() throws Exception { + isRemotePublicationEnabled = false; + prepareCluster(3, 2, Settings.EMPTY); + ensureStableCluster(5); + // enable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, true).build()) + .get(); + interceptTransportActions(PUBLISH_REMOTE_STATE_ACTION_NAME, PUBLISH_STATE_ACTION_NAME); + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 2)); + ensureGreen(INDEX_NAME); + + assertEquals(0, (int) getRequestCount(PUBLISH_STATE_ACTION_NAME)); + assertTrue(getRequestCount(PUBLISH_REMOTE_STATE_ACTION_NAME) > 0); + + // disable remote publication + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_PUBLICATION_SETTING_KEY, false).build()) + .get(); + + internalCluster().fullRestart(); + + ensureStableCluster(5); + NodesStatsResponse response = internalCluster().client() + .admin() + .cluster() + .prepareNodesStats(internalCluster().getNodeNames()) + .clear() + .addMetric(DISCOVERY.metricName()) + .get(); + response.getNodes().forEach(nodeStats -> { + DiscoveryStats discoveryStats = nodeStats.getDiscoveryStats(); + assertNotNull(discoveryStats.getClusterStateStats()); + // ensure none of the nodes received remote publication + discoveryStats.getClusterStateStats() + .getPersistenceStats() + .stream() + .filter( + persistedStateStats -> persistedStateStats.getStatsName().equals(FULL_DOWNLOAD_STATS) + || persistedStateStats.getStatsName().equals(DIFF_DOWNLOAD_STATS) + ) + .forEach(persistedStateStats -> { + assertEquals(0, persistedStateStats.getSuccessCount()); + assertEquals(0, persistedStateStats.getFailedCount()); + assertEquals(0, persistedStateStats.getTotalTimeInMillis()); + }); + }); + } + + private void assertDataNodeDownloadStats(NodeStats nodeStats) { // assert cluster state stats for data node - DiscoveryStats dataNodeDiscoveryStats = nodesStatsResponse.getNodes().get(0).getDiscoveryStats(); + DiscoveryStats dataNodeDiscoveryStats = nodeStats.getDiscoveryStats(); assertNotNull(dataNodeDiscoveryStats.getClusterStateStats()); assertEquals(0, dataNodeDiscoveryStats.getClusterStateStats().getUpdateSuccess()); assertTrue(dataNodeDiscoveryStats.getClusterStateStats().getPersistenceStats().get(0).getSuccessCount() > 0); @@ -446,4 +629,94 @@ private Map getMetadataFiles(BlobStoreRepository repository, St return fileName.split(DELIMITER)[0]; }).collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum)); } + + private static void clearInterceptedActions() { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get().instance + .clearInterceptedActions(); + } + } + + private static void interceptTransportActions(String... actions) { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get().instance + .interceptTransportActions(actions); + } + } + + private static Integer getRequestCount(String action) { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + return pluginsServices.iterator() + .next() + .filterPlugins(InterceptingTransportService.TestPlugin.class) + .stream() + .findFirst() + .get().instance.getRequestCount(action); + } + + public static class InterceptingTransportService implements TransportInterceptor { + + public static class TestPlugin extends Plugin implements NetworkPlugin { + public final InterceptingTransportService instance = new InterceptingTransportService(); + + @Override + public List getTransportInterceptors( + NamedWriteableRegistry namedWriteableRegistry, + ThreadContext threadContext + ) { + return Collections.singletonList(instance); + } + } + + private final Set actions = new HashSet<>(); + private final Map requests = new HashMap<>(); + + @Override + public TransportRequestHandler interceptHandler( + String action, + String executor, + boolean forceExecution, + TransportRequestHandler actualHandler + ) { + return new InterceptingTransportService.InterceptingHandler<>(action, actualHandler); + } + + synchronized Integer getRequestCount(String action) { + return requests.getOrDefault(action, 0); + } + + synchronized void interceptTransportActions(String... actions) { + Collections.addAll(this.actions, actions); + } + + synchronized void clearInterceptedActions() { + actions.clear(); + requests.clear(); + } + + private class InterceptingHandler implements TransportRequestHandler { + private final String action; + private final TransportRequestHandler handler; + private final Logger logger = LogManager.getLogger(InterceptingHandler.class); + + InterceptingHandler(String action, TransportRequestHandler handler) { + this.action = action; + this.handler = handler; + } + + @Override + public void messageReceived(T request, TransportChannel channel, Task task) throws Exception { + synchronized (this) { + if (actions.contains(action)) { + Integer requestCount = requests.getOrDefault(action, 0); + requests.put(action, requestCount + 1); + logger.info("intercepted action: {} count: {}", action, requestCount + 1); + } + } + handler.messageReceived(request, channel, task); + } + } + } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 9cffc7051d756..01b02db20fb24 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -53,7 +53,6 @@ import java.util.Set; import static org.opensearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -81,7 +80,6 @@ public class CoordinationState { private VotingConfiguration lastPublishedConfiguration; private VoteCollection publishVotes; private final boolean isRemoteStateEnabled; - private boolean isRemotePublicationEnabled; public CoordinationState( DiscoveryNode localNode, @@ -105,14 +103,6 @@ public CoordinationState( .getLastAcceptedConfiguration(); this.publishVotes = new VoteCollection(); this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); - // ToDo: revisit this check while making the setting dynamic - this.isRemotePublicationEnabled = isRemoteStateEnabled - && REMOTE_PUBLICATION_SETTING.get(settings) - && localNode.isRemoteStatePublicationEnabled(); - } - - public boolean isRemotePublicationEnabled() { - return isRemotePublicationEnabled; } public long getCurrentTerm() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 02d5f8431f0ad..684a6b0c3eae5 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -188,9 +188,10 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private Optional currentPublication = Optional.empty(); private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; + private final RemoteClusterStateService remoteClusterStateService; private final RemoteStoreNodeService remoteStoreNodeService; private NodeConnectionsService nodeConnectionsService; - private final RemoteClusterStateService remoteClusterStateService; + private final ClusterSettings clusterSettings; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -315,6 +316,7 @@ public Coordinator( this.localNodeCommissioned = true; this.remoteStoreNodeService = remoteStoreNodeService; this.remoteClusterStateService = remoteClusterStateService; + this.clusterSettings = clusterSettings; } private ClusterFormationState getClusterFormationState() { @@ -1364,7 +1366,7 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( clusterChangedEvent, - coordinationState.get().isRemotePublicationEnabled(), + this.isRemotePublicationEnabled(), persistedStateRegistry ); logger.debug("initialized PublicationContext using class: {}", publicationContext.getClass().toString()); @@ -1893,8 +1895,8 @@ public static boolean isZen1Node(DiscoveryNode discoveryNode) { } public boolean isRemotePublicationEnabled() { - if (coordinationState.get() != null) { - return coordinationState.get().isRemotePublicationEnabled(); + if (remoteClusterStateService != null) { + return remoteClusterStateService.isRemotePublicationEnabled(); } return false; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index 220093b428989..ea8f980c14972 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -49,7 +49,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableConfigured; /** * A Service which provides APIs to upload and download routing table from remote store. @@ -76,7 +76,7 @@ public InternalRemoteRoutingTableService( ThreadPool threadpool, String clusterName ) { - assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; + assert isRemoteRoutingTableConfigured(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; this.threadPool = threadpool; @@ -234,7 +234,7 @@ protected void doClose() throws IOException { @Override protected void doStart() { - assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; + assert isRemoteRoutingTableConfigured(settings) == true : "Remote routing table is not enabled"; final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY ); diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java index 56dfa03215a64..4fc616f521eab 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceFactory.java @@ -15,7 +15,7 @@ import java.util.function.Supplier; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableConfigured; /** * Factory to provide impl for RemoteRoutingTableService based on settings. @@ -37,7 +37,7 @@ public static RemoteRoutingTableService getService( ThreadPool threadPool, String clusterName ) { - if (isRemoteRoutingTableEnabled(settings)) { + if (isRemoteRoutingTableConfigured(settings)) { return new InternalRemoteRoutingTableService(repositoriesService, settings, clusterSettings, threadPool, clusterName); } return new NoopRemoteRoutingTableService(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 1d7200792442f..0cd2025b98783 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -83,6 +83,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.LongSupplier; @@ -112,6 +113,8 @@ import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable.INDEX_ROUTING_METADATA_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteClusterStateConfigured; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableConfigured; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** * A Service which provides APIs to upload and download cluster metadata from remote store. @@ -132,7 +135,7 @@ public class RemoteClusterStateService implements Closeable { REMOTE_PUBLICATION_SETTING_KEY, false, Property.NodeScope, - Property.Final + Property.Dynamic ); /** @@ -232,7 +235,7 @@ public static RemoteClusterStateValidationMode parseString(String mode) { private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]"; - private final boolean isPublicationEnabled; + private volatile AtomicBoolean isPublicationEnabled; private final String remotePathPrefix; private final RemoteClusterStateCache remoteClusterStateCache; @@ -273,9 +276,12 @@ public RemoteClusterStateService( this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; this.indexMetadataUploadListeners = indexMetadataUploadListeners; - this.isPublicationEnabled = REMOTE_PUBLICATION_SETTING.get(settings) - && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings) - && RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings); + this.isPublicationEnabled = new AtomicBoolean( + clusterSettings.get(REMOTE_PUBLICATION_SETTING) + && RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings) + && RemoteStoreNodeAttribute.isRemoteRoutingTableConfigured(settings) + ); + clusterSettings.addSettingsUpdateConsumer(REMOTE_PUBLICATION_SETTING, this::setRemotePublicationSetting); this.remotePathPrefix = CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.get(settings); this.remoteRoutingTableService = RemoteRoutingTableServiceFactory.getService( repositoriesService, @@ -303,19 +309,20 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat return null; } + boolean publicationEnabled = isPublicationEnabled.get(); UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, new ArrayList<>(clusterState.metadata().indices().values()), emptyMap(), - RemoteGlobalMetadataManager.filterCustoms(clusterState.metadata().customs(), isPublicationEnabled), + RemoteGlobalMetadataManager.filterCustoms(clusterState.metadata().customs(), publicationEnabled), true, true, true, - isPublicationEnabled, - isPublicationEnabled, - isPublicationEnabled, - isPublicationEnabled ? clusterState.customs() : Collections.emptyMap(), - isPublicationEnabled, + publicationEnabled, + publicationEnabled, + publicationEnabled, + publicationEnabled ? clusterState.customs() : Collections.emptyMap(), + publicationEnabled, remoteRoutingTableService.getIndicesRouting(clusterState.getRoutingTable()), null ); @@ -394,9 +401,9 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles(); final DiffableUtils.MapDiff> customsDiff = remoteGlobalMetadataManager - .getCustomsDiff(clusterState, previousClusterState, firstUploadForSplitGlobalMetadata, isPublicationEnabled); + .getCustomsDiff(clusterState, previousClusterState, firstUploadForSplitGlobalMetadata, isPublicationEnabled.get()); final DiffableUtils.MapDiff> clusterStateCustomsDiff = - remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, previousClusterState, isPublicationEnabled, false); + remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, previousClusterState, isPublicationEnabled.get(), false); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); final Map allUploadedClusterStateCustomsMap = new HashMap<>( previousManifest.getClusterStateCustomMap() @@ -461,10 +468,10 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; - final boolean updateDiscoveryNodes = isPublicationEnabled + final boolean updateDiscoveryNodes = isPublicationEnabled.get() && clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges(); - final boolean updateClusterBlocks = isPublicationEnabled && !clusterState.blocks().equals(previousClusterState.blocks()); - final boolean updateHashesOfConsistentSettings = isPublicationEnabled + final boolean updateClusterBlocks = isPublicationEnabled.get() && !clusterState.blocks().equals(previousClusterState.blocks()); + final boolean updateHashesOfConsistentSettings = isPublicationEnabled.get() && Metadata.isHashesOfConsistentSettingsEqual(previousClusterState.metadata(), clusterState.metadata()) == false; uploadedMetadataResults = writeMetadataInParallel( @@ -1115,6 +1122,14 @@ private void setChecksumValidationMode(RemoteClusterStateValidationMode remoteCl this.remoteClusterStateValidationMode = remoteClusterStateValidationMode; } + private void setRemotePublicationSetting(boolean remotePublicationSetting) { + if (remotePublicationSetting == false) { + this.isPublicationEnabled.set(false); + } else { + this.isPublicationEnabled.set(isRemoteStoreClusterStateEnabled(settings) && isRemoteRoutingTableConfigured(settings)); + } + } + // Package private for unit test RemoteRoutingTableService getRemoteRoutingTableService() { return this.remoteRoutingTableService; @@ -1830,7 +1845,7 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } public boolean isRemotePublicationEnabled() { - return this.isPublicationEnabled; + return this.isPublicationEnabled.get(); } public void setRemoteStateReadTimeout(TimeValue remoteStateReadTimeout) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 11f26ac8b3ed9..1a8e85f30527d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -21,8 +21,8 @@ public class RemotePersistenceStats { RemoteDownloadStats remoteDiffDownloadStats; RemoteDownloadStats remoteFullDownloadStats; - final String FULL_DOWNLOAD_STATS = "remote_full_download"; - final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; + public static final String FULL_DOWNLOAD_STATS = "remote_full_download"; + public static final String DIFF_DOWNLOAD_STATS = "remote_diff_download"; public RemotePersistenceStats() { remoteUploadStats = new RemoteUploadStats(); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index d6a58f8e1d471..d52b37f9a7bd6 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -202,7 +202,7 @@ private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { .isEmpty() == false; } - public static boolean isRemoteRoutingTableEnabled(Settings settings) { + public static boolean isRemoteRoutingTableConfigured(Settings settings) { return isRemoteRoutingTableAttributePresent(settings); } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 32cb95e0c04f6..b5d16e7be849f 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -68,7 +68,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -1268,16 +1267,6 @@ public void testHandleCommitOnFollowerNodeWhenRemotePublicationEnabledWithNullRe verifyNoInteractions(remoteClusterStateService); } - public void testIsRemotePublicationEnabled_WithInconsistentSettings() { - // create settings with remote state disabled but publication enabled - Settings settings = Settings.builder() - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), false) - .put(REMOTE_PUBLICATION_SETTING_KEY, true) - .build(); - CoordinationState coordinationState = createCoordinationState(psr1, node1, settings); - assertFalse(coordinationState.isRemotePublicationEnabled()); - } - public static CoordinationState createCoordinationState( PersistedStateRegistry persistedStateRegistry, DiscoveryNode localNode, diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 35a8ae16cacf7..dffbb9d82545a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -151,7 +151,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableConfigured; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -370,6 +370,8 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException // TODO Make the publication flag parameterized publicationEnabled = true; settings = Settings.builder().put(settings).put(REMOTE_PUBLICATION_SETTING_KEY, publicationEnabled).build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, @@ -388,6 +390,7 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException ), writableRegistry() ); + assertTrue(remoteClusterStateService.isRemotePublicationEnabled()); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()) .customs( Map.of( @@ -747,6 +750,8 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws IOException { publicationEnabled = true; settings = Settings.builder().put(settings).put(REMOTE_PUBLICATION_SETTING_KEY, true).build(); + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); remoteClusterStateService = new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, @@ -765,6 +770,7 @@ public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws I ), writableRegistry() ); + assertTrue(remoteClusterStateService.isRemotePublicationEnabled()); final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); @@ -2749,7 +2755,7 @@ public void testRemoteStateUploadStats() throws IOException { } public void testRemoteRoutingTableNotInitializedWhenDisabled() { - if (isRemoteRoutingTableEnabled(settings)) { + if (isRemoteRoutingTableConfigured(settings)) { assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof InternalRemoteRoutingTableService); } else { assertTrue(remoteClusterStateService.getRemoteRoutingTableService() instanceof NoopRemoteRoutingTableService); From dc8a435f9f14eb6eb679c63d06f4baca3def3215 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Thu, 17 Oct 2024 16:08:01 +0530 Subject: [PATCH 250/357] [Star tree] Performance optimizations during flush flow (#16037) --------- Signed-off-by: Bharathwaj G --- .../common/util/ByteArrayBackedBitset.java | 19 +--- .../builder/AbstractDocumentsFileManager.java | 106 ++++++++++++++---- .../startree/builder/BaseStarTreeBuilder.java | 49 ++++---- .../builder/OffHeapStarTreeBuilder.java | 41 ++++++- .../utils/StarTreeDocumentBitSetUtil.java | 32 +++--- .../util/ByteArrayBackedBitsetTests.java | 7 +- .../StarTreeDocumentBitSetUtilTests.java | 12 +- 7 files changed, 184 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/ByteArrayBackedBitset.java b/server/src/main/java/org/opensearch/common/util/ByteArrayBackedBitset.java index 2d7948d414937..a0c14ac8e9116 100644 --- a/server/src/main/java/org/opensearch/common/util/ByteArrayBackedBitset.java +++ b/server/src/main/java/org/opensearch/common/util/ByteArrayBackedBitset.java @@ -8,11 +8,10 @@ package org.opensearch.common.util; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; import java.io.IOException; +import java.nio.ByteBuffer; /** * A bitset backed by a byte array. This will initialize and set bits in the byte array based on the index. @@ -39,18 +38,6 @@ public ByteArrayBackedBitset(RandomAccessInput in, long offset, int length) thro } } - /** - * Constructor which set the Lucene's IndexInput to read the bitset into a read-only buffer. - */ - public ByteArrayBackedBitset(IndexInput in, int length) throws IOException { - byteArray = new byte[length]; - int i = 0; - while (i < length) { - byteArray[i] = in.readByte(); - i++; - } - } - /** * Sets the bit at the given index to 1. * Each byte can indicate 8 bits, so the index is divided by 8 to get the byte array index. @@ -61,10 +48,10 @@ public void set(int index) { byteArray[byteArrIndex] |= (byte) (1 << (index & 7)); } - public int write(IndexOutput output) throws IOException { + public int write(ByteBuffer output) throws IOException { int numBytes = 0; for (Byte bitSet : byteArray) { - output.writeByte(bitSet); + output.put(bitSet); numBytes += Byte.BYTES; } return numBytes; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java index 3f4e302e0a0f2..e91505d180105 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java @@ -16,6 +16,8 @@ import org.apache.lucene.store.TrackingDirectoryWrapper; import org.apache.lucene.util.NumericUtils; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; @@ -24,6 +26,8 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.List; import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.DOUBLE; @@ -67,54 +71,84 @@ private void setDocSizeInBytes(int numBytes) { } /** - * Write the star tree document to file associated with dimensions and metrics + * Write the star tree document to a byte buffer */ protected int writeStarTreeDocument(StarTreeDocument starTreeDocument, IndexOutput output, boolean isAggregatedDoc) throws IOException { - int numBytes = writeDimensions(starTreeDocument, output); - numBytes += writeMetrics(starTreeDocument, output, isAggregatedDoc); + int numBytes = calculateDocumentSize(starTreeDocument, isAggregatedDoc); + byte[] bytes = new byte[numBytes]; + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()); + writeDimensions(starTreeDocument, buffer); + if (isAggregatedDoc == false) { + writeFlushMetrics(starTreeDocument, buffer); + } else { + writeMetrics(starTreeDocument, buffer, isAggregatedDoc); + } + output.writeBytes(bytes, bytes.length); setDocSizeInBytes(numBytes); - return numBytes; + return bytes.length; } /** - * Write dimensions to file + * Write dimensions to the byte buffer */ - protected int writeDimensions(StarTreeDocument starTreeDocument, IndexOutput output) throws IOException { - int numBytes = 0; - for (int i = 0; i < starTreeDocument.dimensions.length; i++) { - output.writeLong(starTreeDocument.dimensions[i] == null ? 0L : starTreeDocument.dimensions[i]); - numBytes += Long.BYTES; + protected void writeDimensions(StarTreeDocument starTreeDocument, ByteBuffer buffer) throws IOException { + for (Long dimension : starTreeDocument.dimensions) { + buffer.putLong(dimension == null ? 0L : dimension); } - numBytes += StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.dimensions, output); - return numBytes; + StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.dimensions, buffer); } /** * Write star tree document metrics to file */ - protected int writeMetrics(StarTreeDocument starTreeDocument, IndexOutput output, boolean isAggregatedDoc) throws IOException { - int numBytes = 0; + protected void writeFlushMetrics(StarTreeDocument starTreeDocument, ByteBuffer buffer) throws IOException { + for (int i = 0; i < starTreeDocument.metrics.length; i++) { + buffer.putLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); + } + StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.metrics, buffer); + } + + /** + * Write star tree document metrics to the byte buffer + */ + protected void writeMetrics(StarTreeDocument starTreeDocument, ByteBuffer buffer, boolean isAggregatedDoc) throws IOException { for (int i = 0; i < starTreeDocument.metrics.length; i++) { FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); if (aggregatedValueType.equals(LONG)) { - output.writeLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); - numBytes += Long.BYTES; + buffer.putLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); } else if (aggregatedValueType.equals(DOUBLE)) { if (isAggregatedDoc) { long val = NumericUtils.doubleToSortableLong( starTreeDocument.metrics[i] == null ? 0.0 : (Double) starTreeDocument.metrics[i] ); - output.writeLong(val); + buffer.putLong(val); } else { - output.writeLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); + buffer.putLong(starTreeDocument.metrics[i] == null ? 0L : (Long) starTreeDocument.metrics[i]); } - numBytes += Long.BYTES; } else { throw new IllegalStateException("Unsupported metric type"); } } - numBytes += StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.metrics, output); - return numBytes; + StarTreeDocumentBitSetUtil.writeBitSet(starTreeDocument.metrics, buffer); + } + + /** + * Calculate the size of the serialized StarTreeDocument + */ + private int calculateDocumentSize(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) { + int size = starTreeDocument.dimensions.length * Long.BYTES; + size += getLength(starTreeDocument.dimensions); + + for (int i = 0; i < starTreeDocument.metrics.length; i++) { + size += Long.BYTES; + } + size += getLength(starTreeDocument.metrics); + + return size; + } + + private static int getLength(Object[] array) { + return (array.length / 8) + (array.length % 8 == 0 ? 0 : 1); } /** @@ -132,7 +166,11 @@ protected StarTreeDocument readStarTreeDocument(RandomAccessInput input, long of offset = readDimensions(dimensions, input, offset); Object[] metrics = new Object[numMetrics]; - offset = readMetrics(input, offset, numMetrics, metrics, isAggregatedDoc); + if (isAggregatedDoc == false) { + offset = readMetrics(input, offset, metrics); + } else { + offset = readMetrics(input, offset, numMetrics, metrics, isAggregatedDoc); + } assert (offset - initialOffset) == docSizeInBytes; return new StarTreeDocument(dimensions, metrics); } @@ -154,10 +192,32 @@ protected long readDimensions(Long[] dimensions, RandomAccessInput input, long o return offset; } + /** + * Read metrics based on metric field values. Then we reuse the metric field values to each of the metric stats. + */ + private long readMetrics(RandomAccessInput input, long offset, Object[] metrics) throws IOException { + Object[] fieldMetrics = new Object[starTreeField.getMetrics().size()]; + for (int i = 0; i < starTreeField.getMetrics().size(); i++) { + fieldMetrics[i] = input.readLong(offset); + offset += Long.BYTES; + } + offset += StarTreeDocumentBitSetUtil.readBitSet(input, offset, fieldMetrics, index -> null); + int fieldIndex = 0; + int numMetrics = 0; + for (Metric metric : starTreeField.getMetrics()) { + for (MetricStat stat : metric.getBaseMetrics()) { + metrics[numMetrics] = fieldMetrics[fieldIndex]; + numMetrics++; + } + fieldIndex++; + } + return offset; + } + /** * Read star tree metrics from file */ - protected long readMetrics(RandomAccessInput input, long offset, int numMetrics, Object[] metrics, boolean isAggregatedDoc) + private long readMetrics(RandomAccessInput input, long offset, int numMetrics, Object[] metrics, boolean isAggregatedDoc) throws IOException { for (int i = 0; i < numMetrics; i++) { FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 2d4938eeb45b3..c3ea04d52e892 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -183,6 +183,7 @@ public List generateMetricAggregatorInfos(MapperService ma /** * Generates the configuration required to perform aggregation for all the metrics on a field + * Each metric field is associated with a metric reader * * @return list of MetricAggregatorInfo */ @@ -191,24 +192,20 @@ public List getMetricReaders(SegmentWriteState stat List metricReaders = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { - for (MetricStat metricStat : metric.getBaseMetrics()) { - SequentialDocValuesIterator metricReader; - FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); - if (metricStat.equals(MetricStat.DOC_COUNT)) { - // _doc_count is numeric field , so we convert to sortedNumericDocValues and get iterator - metricReader = getIteratorForNumericField(fieldProducerMap, metricFieldInfo, DocCountFieldMapper.NAME); - } else { - if (metricFieldInfo == null) { - metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); - } - metricReader = new SequentialDocValuesIterator( - new SortedNumericStarTreeValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) - ) - ); + SequentialDocValuesIterator metricReader; + FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); + if (metric.getField().equals(DocCountFieldMapper.NAME)) { + metricReader = getIteratorForNumericField(fieldProducerMap, metricFieldInfo, DocCountFieldMapper.NAME); + } else { + if (metric.getBaseMetrics().isEmpty()) continue; + if (metricFieldInfo == null) { + metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); } - metricReaders.add(metricReader); + metricReader = new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo)) + ); } + metricReaders.add(metricReader); } return metricReaders; } @@ -572,11 +569,20 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte */ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List metricsReaders) throws IOException { Object[] metrics = new Object[numMetrics]; - for (int i = 0; i < numMetrics; i++) { - SequentialDocValuesIterator metricStatReader = metricsReaders.get(i); - if (metricStatReader != null) { + int metricIndex = 0; + for (int i = 0; i < starTreeField.getMetrics().size(); i++) { + Metric metric = starTreeField.getMetrics().get(i); + if (metric.getBaseMetrics().isEmpty()) continue; + SequentialDocValuesIterator metricReader = metricsReaders.get(i); + if (metricReader != null) { try { - metricStatReader.nextEntry(currentDocId); + metricReader.nextEntry(currentDocId); + Object metricValue = metricReader.value(currentDocId); + + for (MetricStat metricStat : metric.getBaseMetrics()) { + metrics[metricIndex] = metricValue; + metricIndex++; + } } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -584,9 +590,8 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId, List sortAndAggregateSegmentDocuments( } try { for (int i = 0; i < totalSegmentDocs; i++) { - StarTreeDocument document = getSegmentStarTreeDocument(i, dimensionReaders, metricReaders); + StarTreeDocument document = getSegmentStarTreeDocumentWithMetricFieldValues(i, dimensionReaders, metricReaders); segmentDocumentFileManager.writeStarTreeDocument(document, false); } } catch (IOException ex) { @@ -128,6 +128,45 @@ public Iterator sortAndAggregateSegmentDocuments( return sortAndReduceDocuments(sortedDocIds, totalSegmentDocs, false); } + /** + * Returns the star-tree document from the segment based on the current doc id + */ + StarTreeDocument getSegmentStarTreeDocumentWithMetricFieldValues( + int currentDocId, + SequentialDocValuesIterator[] dimensionReaders, + List metricReaders + ) throws IOException { + Long[] dimensions = getStarTreeDimensionsFromSegment(currentDocId, dimensionReaders); + Object[] metricValues = getStarTreeMetricFieldValuesFromSegment(currentDocId, metricReaders); + return new StarTreeDocument(dimensions, metricValues); + } + + /** + * Returns the metric field values for the star-tree document from the segment based on the current doc id + */ + private Object[] getStarTreeMetricFieldValuesFromSegment(int currentDocId, List metricReaders) { + Object[] metricValues = new Object[starTreeField.getMetrics().size()]; + for (int i = 0; i < starTreeField.getMetrics().size(); i++) { + if (starTreeField.getMetrics().get(i).getBaseMetrics().isEmpty()) continue; + SequentialDocValuesIterator metricReader = metricReaders.get(i); + if (metricReader != null) { + try { + metricReader.nextEntry(currentDocId); + metricValues[i] = metricReader.value(currentDocId); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + throw new RuntimeException("unable to iterate to next doc", e); + } catch (Exception e) { + logger.error("unable to read the metric values from the segment", e); + throw new IllegalStateException("unable to read the metric values from the segment", e); + } + } else { + throw new IllegalStateException("metric reader is empty"); + } + } + return metricValues; + } + /** * Sorts and aggregates the star-tree documents from multiple segments and builds star tree based on the newly * aggregated star-tree documents diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtil.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtil.java index a508e497adcdf..4e7ec30f23c3b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtil.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtil.java @@ -8,34 +8,17 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; import org.opensearch.common.util.ByteArrayBackedBitset; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.function.Function; /** * Helper class to read/write bitset for null values and identity values. */ public class StarTreeDocumentBitSetUtil { - /** - * Write bitset for null values. - * - * @param array array of objects - * @param output output stream - * @return number of bytes written - * @throws IOException if an I/O error occurs while writing to the output stream - */ - public static int writeBitSet(Object[] array, IndexOutput output) throws IOException { - ByteArrayBackedBitset bitset = new ByteArrayBackedBitset(getLength(array)); - for (int i = 0; i < array.length; i++) { - if (array[i] == null) { - bitset.set(i); - } - } - return bitset.write(output); - } /** * Set identity values based on bitset. @@ -51,6 +34,19 @@ public static int readBitSet(RandomAccessInput input, long offset, Object[] arra return bitset.getCurrBytesRead(); } + /** + * Write the bitset for the given array to the ByteBuffer + */ + public static void writeBitSet(Object[] array, ByteBuffer buffer) throws IOException { + ByteArrayBackedBitset bitset = new ByteArrayBackedBitset(getLength(array)); + for (int i = 0; i < array.length; i++) { + if (array[i] == null) { + bitset.set(i); + } + } + bitset.write(buffer); + } + private static int getLength(Object[] array) { return (array.length / 8) + (array.length % 8 == 0 ? 0 : 1); } diff --git a/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java b/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java index 6750a9e48f033..f07c363372333 100644 --- a/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java +++ b/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java @@ -16,6 +16,8 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Path; /** @@ -39,7 +41,10 @@ private static void testWriteAndReadBitset(int randomArraySize, int randomIndex1 IndexOutput indexOutput = fsDirectory.createOutput(TEST_FILE, IOContext.DEFAULT); bitset.set(randomIndex1); bitset.set(randomIndex2); - bitset.write(indexOutput); + byte[] bytes = new byte[randomArraySize]; + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()); + bitset.write(buffer); + indexOutput.writeBytes(bytes, bytes.length); indexOutput.close(); IndexInput in = fsDirectory.openInput(TEST_FILE, IOContext.DEFAULT); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtilTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtilTests.java index 7d1bd37246fae..6c9923898b729 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtilTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDocumentBitSetUtilTests.java @@ -16,6 +16,8 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Path; import java.util.Arrays; import java.util.function.Function; @@ -46,7 +48,11 @@ void testNullBasedOnBitset(Long[] dims) throws IOException { FSDirectory fsDirectory = FSDirectory.open(basePath); String TEST_FILE = "test_file"; IndexOutput indexOutput = fsDirectory.createOutput(TEST_FILE, IOContext.DEFAULT); - StarTreeDocumentBitSetUtil.writeBitSet(dims, indexOutput); + int numBytes = getLength(dims); + byte[] bytes = new byte[numBytes]; + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()); + StarTreeDocumentBitSetUtil.writeBitSet(dims, buffer); + indexOutput.writeBytes(bytes, numBytes); indexOutput.close(); // test null value on read @@ -69,4 +75,8 @@ void testNullBasedOnBitset(Long[] dims) throws IOException { assertEquals(randomLong, (long) dims1[randomNullIndex2]); in.close(); } + + private static int getLength(Object[] array) { + return (array.length / 8) + (array.length % 8 == 0 ? 0 : 1); + } } From e360ceb71c6be63f1f8658338b1d29686115f635 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 17 Oct 2024 11:46:42 -0500 Subject: [PATCH 251/357] Skip media type parsing for known string values (#16358) Signed-off-by: Andrew Ross --- .../core/xcontent/MediaTypeRegistry.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java index bbb55204712d1..82cc3951be753 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -57,6 +57,7 @@ public final class MediaTypeRegistry { private static Map formatToMediaType = Map.of(); private static Map typeWithSubtypeToMediaType = Map.of(); + private static Map knownStringsToMediaType = Map.of(); // Default mediaType singleton private static MediaType DEFAULT_MEDIA_TYPE; @@ -84,6 +85,8 @@ private static void register(MediaType[] acceptedMediaTypes, Map typeMap = new HashMap<>(typeWithSubtypeToMediaType); Map formatMap = new HashMap<>(formatToMediaType); + Map knownStringMap = new HashMap<>(knownStringsToMediaType); + for (MediaType mediaType : acceptedMediaTypes) { if (formatMap.containsKey(mediaType.format())) { throw new IllegalArgumentException("unable to register mediaType: [" + mediaType.format() + "]. Type already exists."); @@ -107,13 +110,24 @@ private static void register(MediaType[] acceptedMediaTypes, Map Date: Thu, 17 Oct 2024 20:39:16 +0300 Subject: [PATCH 252/357] Fixed assignment to catch block parameter (#15384) Signed-off-by: Dmitry Kryukov --- .../java/org/opensearch/search/SearchService.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 40afdbfbdaa9e..c2f8b17fcf166 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -721,14 +721,15 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh } } catch (Exception e) { // execution exception can happen while loading the cache, strip it - if (e instanceof ExecutionException) { - e = (e.getCause() == null || e.getCause() instanceof Exception) - ? (Exception) e.getCause() - : new OpenSearchException(e.getCause()); + Exception exception = e; + if (exception instanceof ExecutionException) { + exception = (exception.getCause() == null || exception.getCause() instanceof Exception) + ? (Exception) exception.getCause() + : new OpenSearchException(exception.getCause()); } - logger.trace("Query phase failed", e); - processFailure(readerContext, e); - throw e; + logger.trace("Query phase failed", exception); + processFailure(readerContext, exception); + throw exception; } finally { taskResourceTrackingService.writeTaskResourceUsage(task, clusterService.localNode().getId()); } From 3b004bfb2aa8377f5f7912e657cd16bd5e525518 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Thu, 17 Oct 2024 17:44:34 -0700 Subject: [PATCH 253/357] Make query groups persistent across process restarts (#16370) * add QueryGroupMetadata to namedXContentWritables Signed-off-by: Kaushal Kumar * add UT Signed-off-by: Kaushal Kumar * apply spotless Signed-off-by: Kaushal Kumar * add CHANGELOG entry Signed-off-by: Kaushal Kumar * improve changelog comment Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kaushal Kumar --- CHANGELOG.md | 1 + .../main/java/org/opensearch/cluster/ClusterModule.java | 7 +++++++ .../java/org/opensearch/cluster/ClusterModuleTests.java | 9 +++++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56b1e7b4ffba3..d7b86fb5a25f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) - Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) +- [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index 66b1740c8fa96..9081432093106 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -327,6 +327,13 @@ public static List getNamedXWriteables() { DecommissionAttributeMetadata::fromXContent ) ); + entries.add( + new NamedXContentRegistry.Entry( + Metadata.Custom.class, + new ParseField(QueryGroupMetadata.TYPE), + QueryGroupMetadata::fromXContent + ) + ); return entries; } diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java index f8240e775cfa5..8d2f2b743af9f 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java @@ -71,6 +71,7 @@ import org.opensearch.common.settings.SettingsModule; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.GatewayAllocator; import org.opensearch.plugins.ClusterPlugin; import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; @@ -331,10 +332,18 @@ public void testRejectsDuplicateExistingShardsAllocatorName() { public void testQueryGroupMetadataRegister() { List customEntries = ClusterModule.getNamedWriteables(); + List customXEntries = ClusterModule.getNamedXWriteables(); assertTrue( customEntries.stream() .anyMatch(entry -> entry.categoryClass == Metadata.Custom.class && entry.name.equals(QueryGroupMetadata.TYPE)) ); + + assertTrue( + customXEntries.stream() + .anyMatch( + entry -> entry.categoryClass == Metadata.Custom.class && entry.name.getPreferredName().equals(QueryGroupMetadata.TYPE) + ) + ); } public void testRerouteServiceSetForBalancedShardsAllocator() { From 4456d559a425271fb35e8d4fb1a000ceb1e2e273 Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Fri, 18 Oct 2024 04:12:09 +0300 Subject: [PATCH 254/357] Removed suspicious call of getClass() on instance of Class, which erased type info (#16002) * Removed suspicious getClass() call on Class Signed-off-by: Dmitry Kryukov * Changed the exception's message. Added unit test. Signed-off-by: Dmitry Kryukov * Run spotless, add license header Signed-off-by: Daniel Widdis --------- Signed-off-by: Dmitry Kryukov Signed-off-by: Daniel Widdis Co-authored-by: Daniel Widdis --- .../core/common/io/stream/Writeable.java | 4 +-- .../core/common/io/stream/WriteableTests.java | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 libs/core/src/test/java/org/opensearch/core/common/io/stream/WriteableTests.java diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java index 960f4bec5eeb5..a46f2bebe3ad2 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/Writeable.java @@ -83,7 +83,7 @@ public static > void registerReader(final byte ordinal, fina public static void registerClassAlias(final Class classInstance, final Class classGeneric) { if (WRITER_CUSTOM_CLASS_MAP.putIfAbsent(classInstance, classGeneric) != null) { - throw new IllegalArgumentException("Streamable custom class already registered [" + classInstance.getClass() + "]"); + throw new IllegalArgumentException("Streamable custom class already registered [" + classInstance.getName() + "]"); } } @@ -96,7 +96,7 @@ public static > W getWriter(final Class clazz) { } /** - * Returns the ristered reader keyed by the unique ordinal + * Returns the registered reader keyed by the unique ordinal */ @SuppressWarnings("unchecked") public static > R getReader(final byte b) { diff --git a/libs/core/src/test/java/org/opensearch/core/common/io/stream/WriteableTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/WriteableTests.java new file mode 100644 index 0000000000000..60ba0b2111f80 --- /dev/null +++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/WriteableTests.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.common.io.stream; + +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.util.concurrent.atomic.AtomicInteger; + +public class WriteableTests extends OpenSearchTestCase { + + public void testRegisterClassAlias() { + Writeable.WriteableRegistry.registerClassAlias(StringBuilder.class, AtomicInteger.class); + try { + Writeable.WriteableRegistry.registerClassAlias(StringBuilder.class, AtomicInteger.class); + Assert.fail("expected exception not thrown"); + } catch (IllegalArgumentException illegalArgumentException) { + Assert.assertEquals( + "Streamable custom class already registered [java.lang.StringBuilder]", + illegalArgumentException.getMessage() + ); + } + } +} From 0bded8836989cb7a0692eeda7e86a253fea7f197 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Fri, 18 Oct 2024 15:32:02 +0800 Subject: [PATCH 255/357] Revert #15258 to figure out a better approach to fix the issue. (#16377) Signed-off-by: zane-neo --- CHANGELOG.md | 1 - .../opensearch/bootstrap/BootstrapTests.java | 42 ------------------- .../org/opensearch/bootstrap/Bootstrap.java | 21 +--------- 3 files changed, 2 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7b86fb5a25f0..8d67ed755fa31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,7 +84,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) -- Fix disk usage exceeds threshold cluster can't spin up issue ([#15258](https://github.com/opensearch-project/OpenSearch/pull/15258))) - [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) diff --git a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java index 7aa63a2736a8c..e9219de218aef 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java @@ -31,7 +31,6 @@ package org.opensearch.bootstrap; -import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.settings.KeyStoreCommandTestCase; import org.opensearch.common.settings.KeyStoreWrapper; import org.opensearch.common.settings.SecureSettings; @@ -39,7 +38,6 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.settings.SecureString; import org.opensearch.env.Environment; -import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; @@ -53,14 +51,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; public class BootstrapTests extends OpenSearchTestCase { Environment env; @@ -139,38 +131,4 @@ private void assertPassphraseRead(String source, String expected) { } } - public void testInitExecutionOrder() throws Exception { - AtomicInteger order = new AtomicInteger(0); - CountDownLatch countDownLatch = new CountDownLatch(1); - Thread mockThread = new Thread(() -> { - assertEquals(0, order.getAndIncrement()); - countDownLatch.countDown(); - }); - - Node mockNode = mock(Node.class); - doAnswer(invocation -> { - try { - boolean threadStarted = countDownLatch.await(1000, TimeUnit.MILLISECONDS); - assertTrue( - "Waited for one second but the keepAliveThread isn't started, please check the execution order of" - + "keepAliveThread.start and node.start", - threadStarted - ); - } catch (InterruptedException e) { - fail("Thread interrupted"); - } - assertEquals(1, order.getAndIncrement()); - return null; - }).when(mockNode).start(); - - LogConfigurator.registerErrorListener(); - Bootstrap testBootstrap = new Bootstrap(mockThread, mockNode); - Bootstrap.setInstance(testBootstrap); - - Bootstrap.startInstance(testBootstrap); - - verify(mockNode).start(); - assertEquals(2, order.get()); - } - } diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 757e2c9da6e49..4e167d10b99fa 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -93,17 +93,6 @@ final class Bootstrap { private final Thread keepAliveThread; private final Spawner spawner = new Spawner(); - // For testing purpose - static void setInstance(Bootstrap bootstrap) { - INSTANCE = bootstrap; - } - - // For testing purpose - Bootstrap(Thread keepAliveThread, Node node) { - this.keepAliveThread = keepAliveThread; - this.node = node; - } - /** creates a new instance */ Bootstrap() { keepAliveThread = new Thread(new Runnable() { @@ -347,10 +336,8 @@ private static Environment createEnvironment( } private void start() throws NodeValidationException { - // keepAliveThread should start first than node to ensure the cluster can spin up successfully in edge cases: - // https://github.com/opensearch-project/OpenSearch/issues/14791 - keepAliveThread.start(); node.start(); + keepAliveThread.start(); } static void stop() throws IOException { @@ -423,7 +410,7 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui throw new BootstrapException(e); } - startInstance(INSTANCE); + INSTANCE.start(); // We don't close stderr if `--quiet` is passed, because that // hides fatal startup errors. For example, if OpenSearch is @@ -475,10 +462,6 @@ static void init(final boolean foreground, final Path pidFile, final boolean qui } } - static void startInstance(Bootstrap instance) throws NodeValidationException { - instance.start(); - } - @SuppressForbidden(reason = "System#out") private static void closeSystOut() { System.out.close(); From 9096aee4fce24e6827b0b758dafb0f1f5d74f224 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:47:16 +0530 Subject: [PATCH 256/357] Fix bug in snapshot update check for multiple v2 repo (#16379) Signed-off-by: Gaurav Bafna --- .../opensearch/repositories/RepositoriesService.java | 12 +++++++++--- .../BlobStoreRepositoryRemoteIndexTests.java | 7 +++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 7da52147661dc..9aec81536dbd0 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -81,6 +81,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -729,7 +730,7 @@ public static void validateRepositoryMetadataSettings( + " in the name as this delimiter is used to create pinning entity" ); } - if (repositoryWithShallowV2Exists(repositories)) { + if (repositoryWithShallowV2Exists(repositories, repositoryName)) { throw new RepositoryException( repositoryName, "setting " @@ -763,8 +764,13 @@ public static void validateRepositoryMetadataSettings( } } - private static boolean repositoryWithShallowV2Exists(Map repositories) { - return repositories.values().stream().anyMatch(repo -> SHALLOW_SNAPSHOT_V2.get(repo.getMetadata().settings())); + private static boolean repositoryWithShallowV2Exists(Map repositories, String repositoryName) { + return repositories.values() + .stream() + .anyMatch( + repository -> SHALLOW_SNAPSHOT_V2.get(repository.getMetadata().settings()) + && !Objects.equals(repository.getMetadata().name(), repositoryName) + ); } private static boolean pinnedTimestampExistsWithDifferentRepository( diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index e280ab8c7a73c..f2b06b0926b81 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -431,6 +431,13 @@ public void testRepositoryCreationShallowV2() throws Exception { ); } + // Modify repo-1 settings. This should go through + updateRepository( + client, + "test-repo-1", + Settings.builder().put(snapshotRepoSettings1).put("max_snapshot_bytes_per_sec", "10k").build() + ); + // Disable shallow snapshot V2 setting on test-repo-1 updateRepository( client, From f1c98a4da0cf6583212eecc9ed8ebc3cd426a918 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:54:30 +0530 Subject: [PATCH 257/357] Do orphan timestamp cleanup before completing the snapshot (#16365) Signed-off-by: Gaurav Bafna --- .../org/opensearch/snapshots/SnapshotsService.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 6e66f8c958666..ac1541b0e607a 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -617,12 +617,12 @@ public void onResponse(RepositoryData repositoryData) { ); return; } - listener.onResponse(snapshotInfo); + cleanOrphanTimestamp(repositoryName, repositoryData); logger.info("created snapshot-v2 [{}] in repository [{}]", repositoryName, snapshotName); + listener.onResponse(snapshotInfo); // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations // can get queued . This is triggering them. runNextQueuedOperation(repositoryData, repositoryName, true); - cleanOrphanTimestamp(repositoryName, repositoryData); } @Override @@ -657,14 +657,8 @@ private void cleanOrphanTimestamp(String repoName, RepositoryData repositoryData if (orphanPinnedEntities.isEmpty()) { return; } - logger.info("Found {} orphan timestamps. Cleaning it up now", orphanPinnedEntities.size()); - if (tryEnterRepoLoop(repoName)) { - deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities); - leaveRepoLoop(repoName); - } else { - logger.info("Concurrent snapshot create/delete is happening. Skipping clean up of orphan timestamps"); - } + deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities); } private boolean isOrphanPinnedEntity(String repoName, Collection snapshotUUIDs, String pinnedEntity) { From f34678836d3ded7a86e459bf22658542858b6f22 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 18 Oct 2024 16:34:57 -0400 Subject: [PATCH 258/357] Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface (#16387) * Add isDualModeEnabled to SecureTransportSettingsProvider interface Signed-off-by: Craig Perkins * Add default impl Signed-off-by: Craig Perkins * Respond to comments, update usages and update docstring Signed-off-by: Craig Perkins * Address feedback Signed-off-by: Craig Perkins * Add ExperimentalApi and add to CHANGELOG Signed-off-by: Craig Perkins * Move DefaultSecureTransportParameters to separate file and add javadoc Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + .../netty4/ssl/SecureNetty4Transport.java | 9 ++++-- .../DefaultSecureTransportParameters.java | 28 +++++++++++++++++++ .../SecureTransportSettingsProvider.java | 18 ++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d67ed755fa31..52333b6a382c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) +- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java index 977121346dcc3..e51ed5663502f 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java @@ -142,9 +142,10 @@ public SSLServerChannelInitializer(String name) { protected void initChannel(Channel ch) throws Exception { super.initChannel(ch); - final boolean dualModeEnabled = NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.get(settings); + final boolean dualModeEnabled = secureTransportSettingsProvider.parameters(settings) + .map(SecureTransportSettingsProvider.SecureTransportParameters::dualModeEnabled) + .orElse(false); if (dualModeEnabled) { - logger.info("SSL Dual mode enabled, using port unification handler"); final ChannelHandler portUnificationHandler = new DualModeSslHandler( settings, secureTransportSettingsProvider, @@ -258,7 +259,9 @@ protected class SSLClientChannelInitializer extends Netty4Transport.ClientChanne public SSLClientChannelInitializer(DiscoveryNode node) { this.node = node; - final boolean dualModeEnabled = NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.get(settings); + final boolean dualModeEnabled = secureTransportSettingsProvider.parameters(settings) + .map(SecureTransportSettingsProvider.SecureTransportParameters::dualModeEnabled) + .orElse(false); hostnameVerificationEnabled = NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION.get(settings); hostnameVerificationResolveHostName = NetworkModule.TRANSPORT_SSL_ENFORCE_HOSTNAME_VERIFICATION_RESOLVE_HOST_NAME.get(settings); diff --git a/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java new file mode 100644 index 0000000000000..e3771f224a7db --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; + +/** + * Default implementation of {@link SecureTransportSettingsProvider.SecureTransportParameters}. + */ +class DefaultSecureTransportParameters implements SecureTransportSettingsProvider.SecureTransportParameters { + private final Settings settings; + + DefaultSecureTransportParameters(Settings settings) { + this.settings = settings; + } + + @Override + public boolean dualModeEnabled() { + return NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.get(settings); + } +} diff --git a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java index 5b7402a01f82d..5f9e1a952b6e8 100644 --- a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java +++ b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java @@ -36,6 +36,24 @@ default Collection> getTransportAdapterProvi return Collections.emptyList(); } + /** + * Returns parameters that can be dynamically provided by a plugin providing a {@link SecureTransportSettingsProvider} + * implementation + * @param settings settings + * @return an instance of {@link SecureTransportParameters} + */ + default Optional parameters(Settings settings) { + return Optional.of(new DefaultSecureTransportParameters(settings)); + } + + /** + * Dynamic parameters that can be provided by the {@link SecureTransportSettingsProvider} + */ + @ExperimentalApi + interface SecureTransportParameters { + boolean dualModeEnabled(); + } + /** * If supported, builds the {@link TransportExceptionHandler} instance for {@link Transport} instance * @param settings settings From 0f7d572da1d2dec4c2b83c6ae2c9e47ce38edd00 Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Sat, 19 Oct 2024 06:08:50 +0300 Subject: [PATCH 259/357] Fix array hashCode calculation in ResyncReplicationRequest (#16378) * Array, passed to Objects.hash(), should be wrapped into Arrays.hashCode(). Added unit test Signed-off-by: Dmitry Kryukov * Updated CHANGELOG.md Signed-off-by: Dmitry Kryukov --------- Signed-off-by: Dmitry Kryukov --- CHANGELOG.md | 2 +- .../resync/ResyncReplicationRequest.java | 2 +- .../resync/ResyncReplicationRequestTests.java | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52333b6a382c7..4415c41d8d1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,8 +86,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) - [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) +- Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) ### Security diff --git a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java index 6a4f2f0607144..a7d6c0abb8705 100644 --- a/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java +++ b/server/src/main/java/org/opensearch/action/resync/ResyncReplicationRequest.java @@ -103,7 +103,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(trimAboveSeqNo, maxSeenAutoIdTimestampOnPrimary, operations); + return Objects.hash(trimAboveSeqNo, maxSeenAutoIdTimestampOnPrimary, Arrays.hashCode(operations)); } @Override diff --git a/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java b/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java index 654dbb203b38a..9faaafc22c844 100644 --- a/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java +++ b/server/src/test/java/org/opensearch/action/resync/ResyncReplicationRequestTests.java @@ -40,14 +40,14 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import static org.hamcrest.Matchers.equalTo; public class ResyncReplicationRequestTests extends OpenSearchTestCase { public void testSerialization() throws IOException { - final byte[] bytes = "{}".getBytes(Charset.forName("UTF-8")); + final byte[] bytes = "{}".getBytes(StandardCharsets.UTF_8); final Translog.Index index = new Translog.Index("id", 0, randomNonNegativeLong(), randomNonNegativeLong(), bytes, null, -1); final ShardId shardId = new ShardId(new Index("index", "uuid"), 0); final ResyncReplicationRequest before = new ResyncReplicationRequest(shardId, 42L, 100, new Translog.Operation[] { index }); @@ -61,4 +61,18 @@ public void testSerialization() throws IOException { assertThat(after, equalTo(before)); } + public void testContractBetweenEqualsAndHashCode() { + final byte[] bytes = "{}".getBytes(StandardCharsets.UTF_8); + final Translog.Index index = new Translog.Index("id", 0, 123L, -123L, bytes, null, -1); + final ShardId shardId = new ShardId(new Index("index", "uuid"), 0); + // Both created requests have arrays `operations` with the same content, and we want to verify that + // equals() and hashCode() are following the contract: + // If objects are equal, they have the same hash code + final ResyncReplicationRequest request1 = new ResyncReplicationRequest(shardId, 42L, 100, new Translog.Operation[] { index }); + final ResyncReplicationRequest request2 = new ResyncReplicationRequest(shardId, 42L, 100, new Translog.Operation[] { index }); + + assertEquals(request1, request2); + assertEquals(request1.hashCode(), request2.hashCode()); + } + } From 0419e5d8a5b5327663c09e93feb931281da7b64e Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Mon, 21 Oct 2024 05:49:28 +0300 Subject: [PATCH 260/357] Fix typo super->sb in method toString() of RemoteStoreNodeAttribute (#15362) * Removed unused StringBuilder Signed-off-by: Dmitry Kryukov * Restored lines, fixed typo super -> sb Signed-off-by: Dmitry Kryukov * Updated CHANGELOG.md Signed-off-by: Dmitry Kryukov --------- Signed-off-by: Dmitry Kryukov --- CHANGELOG.md | 1 + .../opensearch/node/remotestore/RemoteStoreNodeAttribute.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4415c41d8d1c2..d4b6b848f5642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) - Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) +- Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) ### Security diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index d52b37f9a7bd6..b1b6259e4ca18 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -287,6 +287,6 @@ public boolean equals(Object o) { public String toString() { StringBuilder sb = new StringBuilder(); sb.append('{').append(this.repositoriesMetadata).append('}'); - return super.toString(); + return sb.toString(); } } From 78d2a4e237d9e2a0a247fd1f8278346ed3b6fd05 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 21 Oct 2024 13:35:16 +0530 Subject: [PATCH 261/357] Treat last fetch timestamp of pinned timestamp as one of the pinned timestamps (#16392) Signed-off-by: Sachin Kale --- .../RestoreShallowSnapshotV2IT.java | 110 +++++++++++++++++- .../store/RemoteSegmentStoreDirectory.java | 10 +- .../RemoteFsTimestampAwareTranslog.java | 6 +- .../RemoteFsTimestampAwareTranslogTests.java | 81 ++++++++----- 4 files changed, 165 insertions(+), 42 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index d532abaa2b0ad..ecb97e79b348e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -927,7 +927,7 @@ public void testContinuousIndexing() throws Exception { int numDocs = randomIntBetween(200, 300); totalDocs += numDocs; try (BackgroundIndexer indexer = new BackgroundIndexer(index, MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) { - int numberOfSnapshots = 5; + int numberOfSnapshots = 2; for (int i = 0; i < numberOfSnapshots; i++) { logger.info("--> waiting for {} docs to be indexed ...", numDocs); long finalTotalDocs1 = totalDocs; @@ -976,4 +976,112 @@ public void testContinuousIndexing() throws Exception { }); } } + + public void testHashedPrefixTranslogMetadataCombination() throws Exception { + Settings settings = Settings.builder() + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(RemoteStoreEnums.PathType.values())) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean()) + .build(); + + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNode(settings); + String index = "test-index"; + String snapshotRepo = "test-restore-snapshot-repo"; + String baseSnapshotName = "snapshot_"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true)); + + Client client = client(); + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + + createIndex(index, indexSettings); + ensureGreen(index); + + RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance( + RemoteStorePinnedTimestampService.class, + primaryNodeName(index) + ); + + remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(randomIntBetween(1, 5))); + RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueSeconds(randomIntBetween(1, 5))); + + long totalDocs = 0; + Map snapshots = new HashMap<>(); + int numDocs = randomIntBetween(200, 300); + totalDocs += numDocs; + try (BackgroundIndexer indexer = new BackgroundIndexer(index, MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) { + int numberOfSnapshots = 2; + for (int i = 0; i < numberOfSnapshots; i++) { + logger.info("--> waiting for {} docs to be indexed ...", numDocs); + long finalTotalDocs1 = totalDocs; + assertBusy(() -> assertEquals(finalTotalDocs1, indexer.totalIndexedDocs()), 120, TimeUnit.SECONDS); + logger.info("--> {} total docs indexed", totalDocs); + String snapshotName = baseSnapshotName + i; + createSnapshot(snapshotRepo, snapshotName, new ArrayList<>()); + snapshots.put(snapshotName, totalDocs); + if (i < numberOfSnapshots - 1) { + numDocs = randomIntBetween(200, 300); + indexer.continueIndexing(numDocs); + totalDocs += numDocs; + } + } + } + + logger.info("Snapshots Status: " + snapshots); + + for (String snapshot : snapshots.keySet()) { + logger.info("Restoring snapshot: {}", snapshot); + + if (randomBoolean()) { + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(index)).get()); + } else { + assertAcked(client().admin().indices().prepareClose(index)); + } + + assertTrue( + internalCluster().client() + .admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings( + Settings.builder() + .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(RemoteStoreEnums.PathType.values())) + .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean()) + ) + .get() + .isAcknowledged() + ); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshot) + .setWaitForCompletion(true) + .setIndices() + .get(); + + assertEquals(RestStatus.OK, restoreSnapshotResponse1.status()); + + // Verify restored index's stats + ensureGreen(TimeValue.timeValueSeconds(60), index); + long finalTotalDocs = totalDocs; + assertBusy(() -> { + Long hits = client().prepareSearch(index) + .setQuery(matchAllQuery()) + .setSize((int) finalTotalDocs) + .storedFields() + .execute() + .actionGet() + .getHits() + .getTotalHits().value; + + assertEquals(snapshots.get(snapshot), hits); + }); + } + } } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 27a78dc3ce2f6..d51fe0643575e 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -39,7 +39,6 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; -import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.threadpool.ThreadPool; @@ -862,9 +861,11 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException Tuple> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps(); + Set pinnedTimestamps = new HashSet<>(pinnedTimestampsState.v2()); + pinnedTimestamps.add(pinnedTimestampsState.v1()); Set implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles( sortedMetadataFileList, - pinnedTimestampsState.v2(), + pinnedTimestamps, metadataFilePinnedTimestampMap, MetadataFilenameUtils::getTimestamp, MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen @@ -897,11 +898,6 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } - // If pinned timestamps are enabled, make sure to not delete last metadata file. - if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { - metadataFilesEligibleToDelete.remove(sortedMetadataFileList.get(0)); - } - List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream() .filter(metadataFile -> allLockFiles.contains(metadataFile) == false) .collect(Collectors.toList()); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 54cbf8ac9a9f8..99153324b8372 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -189,7 +189,7 @@ public void onResponse(List blobMetadata) { List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted); // If index is not deleted, make sure to keep latest metadata file - if (indexDeleted == false || RemoteStoreSettings.isPinnedTimestampsEnabled()) { + if (indexDeleted == false) { metadataFilesToBeDeleted.remove(metadataFiles.get(0)); } @@ -345,9 +345,11 @@ protected static List getMetadataFilesToBeDeleted( ); // Get md files matching pinned timestamps + Set pinnedTimestamps = new HashSet<>(pinnedTimestampsState.v2()); + pinnedTimestamps.add(pinnedTimestampsState.v1()); Set implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles( metadataFilesToBeDeleted, - pinnedTimestampsState.v2(), + pinnedTimestamps, metadataFilePinnedTimestampMap, file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]), TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 838f97ade9e8e..78ae90936d78e 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -755,17 +755,21 @@ public void testGetGenerationsToBeDeletedWithGenerationInRemote() throws IOExcep assertTrue(generations.isEmpty()); } - public void testGetMetadataFilesToBeDeletedNoExclusion() { + public void testGetMetadataFilesToBeDeletedExclusionDueToRefreshTimestamp() { updatePinnedTimstampTask.run(); - List metadataFiles = List.of( - "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1", - "metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1", - "metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1" - ); + List metadataFiles = new ArrayList<>(); + metadataFiles.add("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1"); + metadataFiles.add("metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1"); + metadataFiles.add("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1"); + // Removing file that is pinned by latest refresh timestamp + List metadataFilesToBeDeleted = new ArrayList<>(metadataFiles); + metadataFilesToBeDeleted.remove( + "metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1" + ); assertEquals( - metadataFiles, + metadataFilesToBeDeleted, RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), Long.MAX_VALUE, false, logger) ); } @@ -774,13 +778,15 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { updatePinnedTimstampTask.run(); long currentTimeInMillis = System.currentTimeMillis(); String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); - String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000); - String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 400000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000); + String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000); List metadataFiles = List.of( - "metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1", - "metadata__9223372036438563903__9223372036854775800__" + md2Timestamp + "__31__9223372036854775803__1", - "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" + "metadata__9223372036438563903__9223372036854774500__" + md1Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854774799__" + md2Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775800__" + md3Timestamp + "__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__" + md4Timestamp + "__31__9223372036854775701__1" ); List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( @@ -791,24 +797,26 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() { logger ); assertEquals(1, metadataFilesToBeDeleted.size()); - assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(1), metadataFilesToBeDeleted.get(0)); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws IOException { long currentTimeInMillis = System.currentTimeMillis(); - String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); - String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); - String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 190000); + String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); + String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); + String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); - long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000; + long pinnedTimestamp = RemoteStoreUtils.invertLong(md3Timestamp) + 10000; when(blobContainer.listBlobs()).thenReturn(Map.of(randomInt(100) + "__" + pinnedTimestamp, new PlainBlobMetadata("xyz", 100))); updatePinnedTimstampTask.run(); List metadataFiles = List.of( - "metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1", - "metadata__9223372036438563903__9223372036854775600__" + md2Timestamp + "__31__9223372036854775803__1", - "metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1" + "metadata__9223372036438563903__9223372036854774500__" + md1Timestamp + "__31__9223372036854775701__1", + "metadata__9223372036438563903__9223372036854774799__" + md2Timestamp + "__31__9223372036854775106__1", + "metadata__9223372036438563903__9223372036854775600__" + md3Timestamp + "__31__9223372036854775803__1", + "metadata__9223372036438563903__9223372036854775701__" + md4Timestamp + "__31__9223372036854775701__1" ); List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( @@ -819,8 +827,8 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws logger ); assertEquals(2, metadataFilesToBeDeleted.size()); - assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); - assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); + assertEquals(metadataFiles.get(1), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(3), metadataFilesToBeDeleted.get(1)); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throws IOException { @@ -856,6 +864,7 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000); String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000); String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000); + String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 800000); when(blobContainer.listBlobs()).thenReturn(Map.of()); @@ -866,8 +875,10 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", // MaxGen 12 "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 9 + "metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1", // MaxGen 10 - "metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1" + "metadata__9223372036438563903__9223372036854775797__" + md4Timestamp + "__31__9223372036854775701__1" ); List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( @@ -878,8 +889,8 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro logger ); assertEquals(2, metadataFilesToBeDeleted.size()); - assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0)); - assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1)); + assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0)); + assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(1)); } public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex() throws IOException { @@ -892,13 +903,15 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex updatePinnedTimstampTask.run(); - List metadataFiles = List.of( - // MaxGen 7 - "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", - // MaxGen 12 - "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", - // MaxGen 17 - "metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1" + List metadataFiles = new ArrayList<>( + List.of( + // MaxGen 12 + "metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1", + // MaxGen 7 + "metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1", + // MaxGen 17 + "metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1" + ) ); List metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted( @@ -908,6 +921,10 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex true, logger ); + + // Metadata file corresponding to latest pinned timestamp fetch is always considered pinned + metadataFiles.remove(metadataFiles.get(2)); + assertEquals(metadataFiles, metadataFilesToBeDeleted); } From c4a9cc1ccbbc49ef19d0f020522e1409505ed718 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:43:23 +0530 Subject: [PATCH 262/357] BugFix - Extract snapshot UUID from pinned entity correctly (#16398) Signed-off-by: Gaurav Bafna --- .../snapshots/SnapshotsService.java | 5 +-- .../snapshots/SnapshotsServiceTests.java | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index ac1541b0e607a..ff1395c600ac0 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -661,7 +661,7 @@ private void cleanOrphanTimestamp(String repoName, RepositoryData repositoryData deleteOrphanTimestamps(pinnedEntities, orphanPinnedEntities); } - private boolean isOrphanPinnedEntity(String repoName, Collection snapshotUUIDs, String pinnedEntity) { + static boolean isOrphanPinnedEntity(String repoName, Collection snapshotUUIDs, String pinnedEntity) { Tuple tokens = getRepoSnapshotUUIDTuple(pinnedEntity); return Objects.equals(tokens.v1(), repoName) && snapshotUUIDs.contains(tokens.v2()) == false; } @@ -748,7 +748,8 @@ public static String getPinningEntity(String repositoryName, String snapshotUUID public static Tuple getRepoSnapshotUUIDTuple(String pinningEntity) { String[] tokens = pinningEntity.split(SNAPSHOT_PINNED_TIMESTAMP_DELIMITER); - return new Tuple<>(tokens[0], tokens[1]); + String snapUUID = String.join(SNAPSHOT_PINNED_TIMESTAMP_DELIMITER, Arrays.copyOfRange(tokens, 1, tokens.length)); + return new Tuple<>(tokens[0], snapUUID); } private void cloneSnapshotPinnedTimestamp( diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java index e374636f60d22..2aa061366465e 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsServiceTests.java @@ -48,6 +48,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; @@ -68,6 +69,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -78,6 +80,7 @@ import org.mockito.ArgumentCaptor; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; +import static org.opensearch.snapshots.SnapshotsService.getRepoSnapshotUUIDTuple; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -804,6 +807,36 @@ public void testRunReadyCloneCompletionListenerFailure() throws Exception { assertEquals(expectedUpdate.hashCode(), capturedUpdate.hashCode()); } + public void testGetRepoSnapshotUUIDTuple() { + String repoName = "repoName"; + String pinningEntity = "repoName__OstrHGrERqaR__-597zHYQ"; + Tuple t = getRepoSnapshotUUIDTuple(pinningEntity); + assertEquals(repoName, t.v1()); + assertEquals("OstrHGrERqaR__-597zHYQ", t.v2()); + } + + public void testIsOrphanPinnedEntity() { + String repoName = "repoName"; + ArrayList snapshotUUIDs = new ArrayList<>( + Arrays.asList("OouZCQ30TqypFBZGgk1C7g", "RSP6GLJfSO6SsMmUjZNAaA", "OstrHGrERqaR__-597zHYQ, Zjlnf8IHRxqFBijj0m52gw") + ); + + ArrayList pinnedEntities = new ArrayList<>( + Arrays.asList( + "repoName__OouZCQ30TqypFBZGgk1C7g", + "repoName__RSP6GLJfSO6SsMmUjZNAaA", + "repoName__OstrHGrERqaR__-597zHYQ, Zjlnf8IHRxqFBijj0m52gw" + ) + ); + + for (String pinnedEntity : pinnedEntities) { + assertFalse(SnapshotsService.isOrphanPinnedEntity(repoName, snapshotUUIDs, pinnedEntity)); + } + + String orphanEntity = "repoName__orphan"; + assertTrue(SnapshotsService.isOrphanPinnedEntity(repoName, snapshotUUIDs, orphanEntity)); + } + /** * Helper method to create a SnapshotsService instance with a provided ClusterService. * This method mocks all necessary dependencies for the SnapshotsService. From ad7f9e79672c8e78eac999293e9cbd09cfa379c9 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 21 Oct 2024 16:00:51 +0530 Subject: [PATCH 263/357] Handle delete cases for star tree (#16380) Signed-off-by: Bharathwaj G --- .../lucene/index/DocValuesProducerUtil.java | 33 ++++++ .../Composite912DocValuesReader.java | 8 +- .../Composite912DocValuesWriter.java | 39 ++++--- .../startree/builder/BaseStarTreeBuilder.java | 1 - .../StarTreeDocValuesFormatTests.java | 100 ++++++++++++++++++ 5 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/index/DocValuesProducerUtil.java diff --git a/server/src/main/java/org/apache/lucene/index/DocValuesProducerUtil.java b/server/src/main/java/org/apache/lucene/index/DocValuesProducerUtil.java new file mode 100644 index 0000000000000..3aebec56b82d9 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/DocValuesProducerUtil.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +import org.apache.lucene.codecs.DocValuesProducer; + +import java.util.Collections; +import java.util.Set; + +/** + * Utility class for DocValuesProducers + * @opensearch.internal + */ +public class DocValuesProducerUtil { + /** + * Returns the segment doc values producers for the given doc values producer. + * If the given doc values producer is not a segment doc values producer, an empty set is returned. + * @param docValuesProducer the doc values producer + * @return the segment doc values producers + */ + public static Set getSegmentDocValuesProducers(DocValuesProducer docValuesProducer) { + if (docValuesProducer instanceof SegmentDocValuesProducer) { + return (((SegmentDocValuesProducer) docValuesProducer).dvProducers); + } + return Collections.emptySet(); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java index bb8a07d856d87..637d3250fda3f 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java @@ -185,7 +185,13 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState // populates the dummy list of field infos to fetch doc id set iterators for respective fields. // the dummy field info is used to fetch the doc id set iterators for respective fields based on field name FieldInfos fieldInfos = new FieldInfos(getFieldInfoList(fields)); - this.readState = new SegmentReadState(readState.directory, readState.segmentInfo, fieldInfos, readState.context); + this.readState = new SegmentReadState( + readState.directory, + readState.segmentInfo, + fieldInfos, + readState.context, + readState.segmentSuffix + ); // initialize star-tree doc values producer diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java index 2225870afae8e..dd35091dece2f 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesProducerUtil; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; @@ -35,7 +36,6 @@ import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.StarTreeMapper; import java.io.IOException; import java.util.ArrayList; @@ -221,12 +221,8 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, } // we have all the required fields to build composite fields if (compositeFieldSet.isEmpty()) { - for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { - if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { - try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { - starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, compositeDocValuesConsumer); - } - } + try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) { + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, compositeDocValuesConsumer); } } } @@ -285,9 +281,20 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { if (mergeState.docValuesProducers[i] instanceof CompositeIndexReader) { reader = (CompositeIndexReader) mergeState.docValuesProducers[i]; } else { - continue; + Set docValuesProducers = DocValuesProducerUtil.getSegmentDocValuesProducers( + mergeState.docValuesProducers[i] + ); + for (DocValuesProducer docValuesProducer : docValuesProducers) { + if (docValuesProducer instanceof CompositeIndexReader) { + reader = (CompositeIndexReader) docValuesProducer; + List compositeFieldInfo = reader.getCompositeIndexFields(); + if (compositeFieldInfo.isEmpty() == false) { + break; + } + } + } } - + if (reader == null) continue; List compositeFieldInfo = reader.getCompositeIndexFields(); for (CompositeIndexFieldInfo fieldInfo : compositeFieldInfo) { if (fieldInfo.getType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) { @@ -295,17 +302,6 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { if (compositeIndexValues instanceof StarTreeValues) { StarTreeValues starTreeValues = (StarTreeValues) compositeIndexValues; List fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), new ArrayList<>()); - if (starTreeField == null) { - starTreeField = starTreeValues.getStarTreeField(); - } - // assert star tree configuration is same across segments - else { - if (starTreeField.equals(starTreeValues.getStarTreeField()) == false) { - throw new IllegalArgumentException( - "star tree field configuration must match the configuration of the field being merged" - ); - } - } fieldsList.add(starTreeValues); starTreeSubsPerField.put(fieldInfo.getField(), fieldsList); } @@ -340,7 +336,8 @@ private static SegmentWriteState getSegmentWriteState(SegmentWriteState segmentW segmentInfo, segmentWriteState.fieldInfos, segmentWriteState.segUpdates, - segmentWriteState.context + segmentWriteState.context, + segmentWriteState.segmentSuffix ); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index c3ea04d52e892..3054e8e66b601 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -735,7 +735,6 @@ private SequentialDocValuesIterator getIteratorForNumericField( * @throws IOException throws an exception if we are unable to add the doc */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { - appendStarTreeDocument(starTreeDocument); numStarTreeDocs++; } diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index 4fe0199f89f41..d35fc6b111c9f 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -15,11 +15,14 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -58,9 +61,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; /** @@ -207,6 +213,100 @@ public void testStarTreeDocValues() throws IOException { directory.close(); } + public void testStarTreeDocValuesWithDeletions() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + + int iterations = 3; + Map map = new HashMap<>(); + List allIds = new ArrayList<>(); + for (int iter = 0; iter < iterations; iter++) { + // Add 10 documents + for (int i = 0; i < 10; i++) { + String id = String.valueOf(random().nextInt() + i); + allIds.add(id); + Document doc = new Document(); + doc.add(new StringField("_id", id, Field.Store.YES)); + int fieldValue = random().nextInt(5) + 1; + doc.add(new SortedNumericDocValuesField("field", fieldValue)); + + int sndvValue = random().nextInt(3); + + doc.add(new SortedNumericDocValuesField("sndv", sndvValue)); + int dvValue = random().nextInt(3); + + doc.add(new SortedNumericDocValuesField("dv", dvValue)); + map.put(sndvValue + "-" + dvValue, fieldValue + map.getOrDefault(sndvValue + "-" + dvValue, 0)); + iw.addDocument(doc); + } + iw.flush(); + } + iw.commit(); + // Delete random number of documents + int docsToDelete = random().nextInt(9); // Delete up to 9 documents + for (int i = 0; i < docsToDelete; i++) { + if (!allIds.isEmpty()) { + String idToDelete = allIds.remove(random().nextInt(allIds.size() - 1)); + iw.deleteDocuments(new Term("_id", idToDelete)); + allIds.remove(idToDelete); + } + } + iw.flush(); + iw.commit(); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = DirectoryReader.open(directory); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Assert star tree documents + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] actualStarTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + for (StarTreeDocument starDoc : actualStarTreeDocuments) { + Long sndvVal = null; + if (starDoc.dimensions[0] != null) { + sndvVal = starDoc.dimensions[0]; + } + Long dvVal = null; + if (starDoc.dimensions[1] != null) { + dvVal = starDoc.dimensions[1]; + } + if (starDoc.metrics[0] != null) { + double metric = (double) starDoc.metrics[0]; + if (map.containsKey(sndvVal + "-" + dvVal)) { + assertEquals((long) map.get(sndvVal + "-" + dvVal), (long) metric); + } + } + } + } + } + ir.close(); + directory.close(); + } + private XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); From 2dfd519890980690072d670176116e032ba5a845 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:33:37 -0400 Subject: [PATCH 264/357] Bump org.jline:jline in /test/fixtures/hdfs-fixture (#16404) Bumps [org.jline:jline](https://github.com/jline/jline3) from 3.27.0 to 3.27.1. - [Release notes](https://github.com/jline/jline3/releases) - [Changelog](https://github.com/jline/jline3/blob/master/changelog.md) - [Commits](https://github.com/jline/jline3/compare/jline-3.27.0...jline-3.27.1) --- updated-dependencies: - dependency-name: org.jline:jline dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 65fb55afbc7bd..2bd0268ca136b 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -77,7 +77,7 @@ dependencies { api "ch.qos.logback:logback-core:1.5.10" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" - api 'org.jline:jline:3.27.0' + api 'org.jline:jline:3.27.1' api 'org.apache.commons:commons-configuration2:2.11.0' api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api ('org.apache.kerby:kerb-admin:2.1.0') { From 6c7581ef6c67abc5bab0431b2be430361d8a7392 Mon Sep 17 00:00:00 2001 From: Lakshya Taragi <157457166+ltaragi@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:46:14 +0530 Subject: [PATCH 265/357] Snapshot _status API: Include in-progress snapshots in total shard count and index filter (#16394) Signed-off-by: Lakshya Taragi --- .../snapshots/SnapshotStatusApisIT.java | 294 +++++++++++++++++- .../status/SnapshotsStatusRequest.java | 17 +- .../TransportSnapshotsStatusAction.java | 235 ++++++++++---- .../cluster/RestSnapshotsStatusAction.java | 3 + 4 files changed, 477 insertions(+), 72 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index 8b6869aa1d81a..123277a3780a2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -619,9 +619,9 @@ public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws ); assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); assertTrue( - exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") + exception.getMessage().contains(" is more than the maximum allowed value of shard count [2] for snapshot status request") ); - }, 1, TimeUnit.MINUTES); + }); // across multiple snapshots assertBusy(() -> { @@ -636,13 +636,13 @@ public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws ); assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); assertTrue( - exception.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request") + exception.getMessage().contains(" is more than the maximum allowed value of shard count [2] for snapshot status request") ); - }, 1, TimeUnit.MINUTES); + }); logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey())); - assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); } public void testSnapshotStatusForIndexFilter() throws Exception { @@ -666,6 +666,7 @@ public void testSnapshotStatusForIndexFilter() throws Exception { String snapshot = "test-snap-1"; createSnapshot(repositoryName, snapshot, List.of(index1, index2, index3)); + // for a completed snapshot assertBusy(() -> { SnapshotStatus snapshotsStatus = client().admin() .cluster() @@ -682,6 +683,96 @@ public void testSnapshotStatusForIndexFilter() throws Exception { }, 1, TimeUnit.MINUTES); } + public void testSnapshotStatusForIndexFilterForInProgressSnapshot() throws Exception { + String repositoryName = "test-repo"; + createRepository(repositoryName, "mock", Settings.builder().put("location", randomRepoPath()).put("block_on_data", true)); + + logger.info("Create indices"); + String index1 = "test-idx-1"; + String index2 = "test-idx-2"; + String index3 = "test-idx-3"; + createIndex(index1, index2, index3); + ensureGreen(); + + logger.info("Indexing some data"); + for (int i = 0; i < 10; i++) { + index(index1, "_doc", Integer.toString(i), "foo", "bar" + i); + index(index2, "_doc", Integer.toString(i), "foo", "baz" + i); + index(index3, "_doc", Integer.toString(i), "foo", "baz" + i); + } + refresh(); + String inProgressSnapshot = "test-in-progress-snapshot"; + + logger.info("Create snapshot"); + ActionFuture createSnapshotResponseActionFuture = startFullSnapshot(repositoryName, inProgressSnapshot); + + logger.info("Block data node"); + waitForBlockOnAnyDataNode(repositoryName, TimeValue.timeValueMinutes(1)); + awaitNumberOfSnapshotsInProgress(1); + + // test normal functioning of index filter for in progress snapshot + assertBusy(() -> { + SnapshotStatus snapshotsStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .setIndices(index1, index2) + .get() + .getSnapshots() + .get(0); + Map snapshotIndexStatusMap = snapshotsStatus.getIndices(); + // Although the snapshot contains 3 indices, the response of status api call only contains results for 2 + assertEquals(snapshotIndexStatusMap.size(), 2); + assertEquals(snapshotIndexStatusMap.keySet(), Set.of(index1, index2)); + }); + + // when a non-existent index is requested in the index-filter + assertBusy(() -> { + // failure due to index not found in snapshot + final String nonExistentIndex1 = "non-existent-index-1"; + final String nonExistentIndex2 = "non-existent-index-2"; + Exception ex = expectThrows( + Exception.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .setIndices(index1, index2, nonExistentIndex1, nonExistentIndex2) + .execute() + .actionGet() + ); + String cause = String.format( + Locale.ROOT, + "indices [%s] missing in snapshot [%s] of repository [%s]", + String.join(", ", List.of(nonExistentIndex2, nonExistentIndex1)), + inProgressSnapshot, + repositoryName + ); + assertEquals(cause, ex.getCause().getMessage()); + + // no error for ignore_unavailable = true and status response contains only the found indices + SnapshotStatus snapshotsStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .setIndices(index1, index2, nonExistentIndex1, nonExistentIndex2) + .setIgnoreUnavailable(true) + .get() + .getSnapshots() + .get(0); + + Map snapshotIndexStatusMap = snapshotsStatus.getIndices(); + assertEquals(snapshotIndexStatusMap.size(), 2); + assertEquals(snapshotIndexStatusMap.keySet(), Set.of(index1, index2)); + }); + + logger.info("Unblock data node"); + unblockAllDataNodes(repositoryName); + + logger.info("Wait for snapshot to finish"); + waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60)); + } + public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { String repositoryName = "test-repo"; String index1 = "test-idx-1"; @@ -705,6 +796,39 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { createSnapshot(repositoryName, snapshot1, List.of(index1, index2, index3)); createSnapshot(repositoryName, snapshot2, List.of(index1)); + assertBusy(() -> { + // failure due to passing index filter for _all value of repository param + Exception ex = expectThrows( + Exception.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus("_all") + .setSnapshots(snapshot1) + .setIndices(index1, index2, index3) + .execute() + .actionGet() + ); + String cause = + "index list filter is supported only when a single 'repository' is passed, but found 'repository' param = [_all]"; + assertTrue(ex.getMessage().contains(cause)); + }); + + assertBusy(() -> { + // failure due to passing index filter for _all value of snapshot param --> gets translated as a blank array + Exception ex = expectThrows( + Exception.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots() + .setIndices(index1, index2, index3) + .execute() + .actionGet() + ); + String cause = "index list filter is supported only when a single 'snapshot' is passed, but found 'snapshot' param = [_all]"; + assertTrue(ex.getMessage().contains(cause)); + }); + assertBusy(() -> { // failure due to passing index filter for multiple snapshots ActionRequestValidationException ex = expectThrows( @@ -717,9 +841,10 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { .execute() .actionGet() ); - String cause = "index list filter is supported only for a single snapshot"; + String cause = + "index list filter is supported only when a single 'snapshot' is passed, but found 'snapshot' param = [[test-snap-1, test-snap-2]]"; assertTrue(ex.getMessage().contains(cause)); - }, 1, TimeUnit.MINUTES); + }); assertBusy(() -> { // failure due to index not found in snapshot @@ -743,7 +868,18 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { ); assertEquals(cause, ex.getCause().getMessage()); - }, 1, TimeUnit.MINUTES); + // no error for ignore_unavailable = true and status response contains only the found indices + SnapshotStatus snapshotsStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot2) + .setIndices(index1, index2, index3) + .setIgnoreUnavailable(true) + .get() + .getSnapshots() + .get(0); + assertEquals(1, snapshotsStatus.getIndices().size()); + }); assertBusy(() -> { // failure due to too many shards requested @@ -763,12 +899,148 @@ public void testSnapshotStatusFailuresWithIndexFilter() throws Exception { .actionGet() ); assertEquals(ex.status(), RestStatus.TOO_MANY_REQUESTS); - assertTrue(ex.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request")); + assertTrue(ex.getMessage().contains(" is more than the maximum allowed value of shard count [2] for snapshot status request")); logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey())); - assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - }, 2, TimeUnit.MINUTES); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + }); + } + + public void testSnapshotStatusShardLimitOfResponseForInProgressSnapshot() throws Exception { + logger.info("Create repository"); + String repositoryName = "test-repo"; + createRepository( + repositoryName, + "mock", + Settings.builder() + .put("location", randomRepoPath()) + .put("compress", false) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put("wait_after_unblock", 200) + ); + + logger.info("Create indices"); + String index1 = "test-idx-1"; + String index2 = "test-idx-2"; + String index3 = "test-idx-3"; + assertAcked(prepareCreate(index1, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); + assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); + assertAcked(prepareCreate(index3, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); + ensureGreen(); + + logger.info("Index some data"); + indexRandomDocs(index1, 10); + indexRandomDocs(index2, 10); + indexRandomDocs(index3, 10); + + logger.info("Create completed snapshot"); + String completedSnapshot = "test-completed-snapshot"; + String blockedNode = blockNodeWithIndex(repositoryName, index1); + client().admin().cluster().prepareCreateSnapshot(repositoryName, completedSnapshot).setWaitForCompletion(false).get(); + waitForBlock(blockedNode, repositoryName, TimeValue.timeValueSeconds(60)); + unblockNode(repositoryName, blockedNode); + waitForCompletion(repositoryName, completedSnapshot, TimeValue.timeValueSeconds(60)); + + logger.info("Index some more data"); + indexRandomDocs(index1, 10); + indexRandomDocs(index2, 10); + indexRandomDocs(index3, 10); + refresh(); + + logger.info("Create in-progress snapshot"); + String inProgressSnapshot = "test-in-progress-snapshot"; + blockedNode = blockNodeWithIndex(repositoryName, index1); + client().admin().cluster().prepareCreateSnapshot(repositoryName, inProgressSnapshot).setWaitForCompletion(false).get(); + waitForBlock(blockedNode, repositoryName, TimeValue.timeValueSeconds(60)); + List snapshotStatuses = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot, completedSnapshot) + .get() + .getSnapshots(); + + assertEquals(2, snapshotStatuses.size()); + assertEquals(SnapshotsInProgress.State.STARTED, snapshotStatuses.get(0).getState()); + assertEquals(SnapshotsInProgress.State.SUCCESS, snapshotStatuses.get(1).getState()); + + logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value"); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 1)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // shard limit exceeded due to inProgress snapshot alone @ without index-filter + assertBusy(() -> { + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .execute() + .actionGet() + ); + assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); + assertTrue( + exception.getMessage().contains(" is more than the maximum allowed value of shard count [1] for snapshot status request") + ); + }); + + // shard limit exceeded due to inProgress snapshot alone @ with index-filter + assertBusy(() -> { + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .setIndices(index1, index2) + .execute() + .actionGet() + ); + assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); + assertTrue( + exception.getMessage().contains(" is more than the maximum allowed value of shard count [1] for snapshot status request") + ); + }); + + logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a slightly higher value"); + updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 5)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // shard limit exceeded due to passing for inProgress but failing for current + completed + + assertBusy(() -> { + SnapshotStatus inProgressSnapshotStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot) + .get() + .getSnapshots() + .get(0); + assertEquals(3, inProgressSnapshotStatus.getShards().size()); + + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, + () -> client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(inProgressSnapshot, completedSnapshot) + .execute() + .actionGet() + ); + assertEquals(exception.status(), RestStatus.TOO_MANY_REQUESTS); + assertTrue( + exception.getMessage().contains(" is more than the maximum allowed value of shard count [5] for snapshot status request") + ); + }); + + unblockNode(repositoryName, blockedNode); + waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60)); + + logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value"); + updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey())); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); } private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java index 3d7fb5b6beb56..a270dcfa53474 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotsStatusRequest.java @@ -41,6 +41,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.Arrays; import static org.opensearch.action.ValidateActions.addValidationError; @@ -124,8 +125,20 @@ public ActionRequestValidationException validate() { if (snapshots == null) { validationException = addValidationError("snapshots is null", validationException); } - if (indices.length != 0 && snapshots.length != 1) { - validationException = addValidationError("index list filter is supported only for a single snapshot", validationException); + if (indices.length != 0) { + if (repository.equals("_all")) { + String error = + "index list filter is supported only when a single 'repository' is passed, but found 'repository' param = [_all]"; + validationException = addValidationError(error, validationException); + } + if (snapshots.length != 1) { + // snapshot param was '_all' (length = 0) or a list of snapshots (length > 1) + String snapshotParamValue = snapshots.length == 0 ? "_all" : Arrays.toString(snapshots); + String error = "index list filter is supported only when a single 'snapshot' is passed, but found 'snapshot' param = [" + + snapshotParamValue + + "]"; + validationException = addValidationError(error, validationException); + } } return validationException; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 8228cb6301c8c..2c8b06bb5e8fe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -99,8 +99,14 @@ public class TransportSnapshotsStatusAction extends TransportClusterManagerNodeA private final TransportNodesSnapshotsStatus transportNodesSnapshotsStatus; + private Set requestedIndexNames; + private long maximumAllowedShardCount; + private int totalShardsRequiredInResponse; + + private boolean requestUsesIndexFilter; + @Inject public TransportSnapshotsStatusAction( TransportService transportService, @@ -145,25 +151,21 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { + setupForRequest(request); + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); List currentSnapshots = SnapshotsService.currentSnapshots( snapshotsInProgress, request.repository(), Arrays.asList(request.snapshots()) ); + if (currentSnapshots.isEmpty()) { buildResponse(snapshotsInProgress, request, currentSnapshots, null, listener); return; } - Set nodesIds = new HashSet<>(); - for (SnapshotsInProgress.Entry entry : currentSnapshots) { - for (final SnapshotsInProgress.ShardSnapshotStatus status : entry.shards().values()) { - if (status.nodeId() != null) { - nodesIds.add(status.nodeId()); - } - } - } + Set nodesIds = getNodeIdsOfCurrentSnapshots(request, currentSnapshots); if (!nodesIds.isEmpty()) { // There are still some snapshots running - check their progress @@ -192,6 +194,97 @@ protected void clusterManagerOperation( } + private void setupForRequest(SnapshotsStatusRequest request) { + requestedIndexNames = new HashSet<>(Arrays.asList(request.indices())); + requestUsesIndexFilter = requestedIndexNames.isEmpty() == false; + totalShardsRequiredInResponse = 0; + maximumAllowedShardCount = clusterService.getClusterSettings().get(MAX_SHARDS_ALLOWED_IN_STATUS_API); + } + + /* + * To get the node IDs of the relevant (according to the index filter) shards which are part of current snapshots + * It also deals with any missing indices (for index-filter case) and calculates the number of shards contributed by all + * the current snapshots to the total count (irrespective of index-filter) + * If this count exceeds the limit, CircuitBreakingException is thrown + * */ + private Set getNodeIdsOfCurrentSnapshots(final SnapshotsStatusRequest request, List currentSnapshots) + throws CircuitBreakingException { + Set nodesIdsOfCurrentSnapshotShards = new HashSet<>(); + int totalShardsAcrossCurrentSnapshots = 0; + + for (SnapshotsInProgress.Entry currentSnapshotEntry : currentSnapshots) { + if (currentSnapshotEntry.remoteStoreIndexShallowCopyV2()) { + // skip current shallow v2 snapshots + continue; + } + if (requestUsesIndexFilter) { + // index-filter is allowed only for a single snapshot, which has to be this one + // first check if any requested indices are missing from this current snapshot + + final Set indicesInCurrentSnapshot = currentSnapshotEntry.indices() + .stream() + .map(IndexId::getName) + .collect(Collectors.toSet()); + + final Set indicesNotFound = requestedIndexNames.stream() + .filter(index -> indicesInCurrentSnapshot.contains(index) == false) + .collect(Collectors.toSet()); + + if (indicesNotFound.isEmpty() == false) { + handleIndexNotFound( + requestedIndexNames, + indicesNotFound, + request, + currentSnapshotEntry.snapshot().getSnapshotId().getName(), + false + ); + } + // the actual no. of shards contributed by this current snapshot will now be calculated + } else { + // all shards of this current snapshot are required in response + totalShardsAcrossCurrentSnapshots += currentSnapshotEntry.shards().size(); + } + + for (final Map.Entry shardStatusEntry : currentSnapshotEntry.shards() + .entrySet()) { + SnapshotsInProgress.ShardSnapshotStatus shardStatus = shardStatusEntry.getValue(); + boolean indexPresentInFilter = requestedIndexNames.contains(shardStatusEntry.getKey().getIndexName()); + + if (requestUsesIndexFilter && indexPresentInFilter) { + // count only those shards whose index belongs to the index-filter + totalShardsAcrossCurrentSnapshots++; + + // for non-index filter case, we already counted all the shards of this current snapshot (non-shallow v2) + } + + if (shardStatus.nodeId() != null) { + if (requestUsesIndexFilter) { + if (indexPresentInFilter) { + // include node only if the index containing this shard belongs to the index filter + nodesIdsOfCurrentSnapshotShards.add(shardStatus.nodeId()); + } + } else { + nodesIdsOfCurrentSnapshotShards.add(shardStatus.nodeId()); + } + } + } + } + + totalShardsRequiredInResponse += totalShardsAcrossCurrentSnapshots; + if (totalShardsRequiredInResponse > maximumAllowedShardCount) { + // index-filter is allowed only for a single snapshot. If index-filter is being used and limit got exceeded, + // this snapshot is current and its relevant indices contribute more shards than the limit + + // if index-filter is not being used and limit got exceed, there could be more shards required in response coming from completed + // snapshots + // but since the limit is already exceeded, we can fail request here + boolean couldInvolveMoreShards = requestUsesIndexFilter == false; + handleMaximumAllowedShardCountExceeded(request.repository(), totalShardsRequiredInResponse, couldInvolveMoreShards); + } + + return nodesIdsOfCurrentSnapshotShards; + } + private void buildResponse( SnapshotsInProgress snapshotsInProgress, SnapshotsStatusRequest request, @@ -215,6 +308,10 @@ private void buildResponse( List shardStatusBuilder = new ArrayList<>(); Map indexIdLookup = null; for (final Map.Entry shardEntry : entry.shards().entrySet()) { + if (requestUsesIndexFilter && requestedIndexNames.contains(shardEntry.getKey().getIndexName()) == false) { + // skip shard if its index does not belong to the index-filter + continue; + } SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.getValue(); if (status.nodeId() != null) { // We should have information about this shard from the shard: @@ -320,7 +417,6 @@ private void loadRepositoryData( String repositoryName, ActionListener listener ) { - maximumAllowedShardCount = clusterService.getClusterSettings().get(MAX_SHARDS_ALLOWED_IN_STATUS_API); final StepListener repositoryDataListener = new StepListener<>(); repositoriesService.getRepositoryData(repositoryName, repositoryDataListener); repositoryDataListener.whenComplete(repositoryData -> { @@ -343,8 +439,7 @@ private void loadRepositoryData( snapshotInfo ); boolean isShallowV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; - long initialSnapshotTotalSize = 0; - if (isShallowV2Snapshot && request.indices().length == 0) { + if (isShallowV2Snapshot && requestUsesIndexFilter == false) { // TODO: add primary store size in bytes at the snapshot level } @@ -430,7 +525,10 @@ private Map snapshotsInfo( .stream() .filter(s -> requestedSnapshotNames.contains(s.getName())) .collect(Collectors.toMap(SnapshotId::getName, Function.identity())); + + // for no index filter-case and excludes shards from shallow v2 snapshots int totalShardsAcrossSnapshots = 0; + for (final String snapshotName : request.snapshots()) { if (currentSnapshotNames.contains(snapshotName)) { // we've already found this snapshot in the current snapshot entries, so skip over @@ -453,23 +551,15 @@ private Map snapshotsInfo( } SnapshotInfo snapshotInfo = snapshot(snapshotsInProgress, repositoryName, snapshotId); boolean isV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; - if (isV2Snapshot == false && request.indices().length == 0) { + if (isV2Snapshot == false && requestUsesIndexFilter == false) { totalShardsAcrossSnapshots += snapshotInfo.totalShards(); } snapshotsInfoMap.put(snapshotId, snapshotInfo); } - if (totalShardsAcrossSnapshots > maximumAllowedShardCount && request.indices().length == 0) { - String message = "[" - + repositoryName - + ":" - + String.join(", ", request.snapshots()) - + "]" - + " Total shard count [" - + totalShardsAcrossSnapshots - + "] is more than the maximum allowed value of shard count [" - + maximumAllowedShardCount - + "] for snapshot status request"; - throw new CircuitBreakingException(message, CircuitBreaker.Durability.PERMANENT); + totalShardsRequiredInResponse += totalShardsAcrossSnapshots; + if (totalShardsRequiredInResponse > maximumAllowedShardCount && requestUsesIndexFilter == false) { + // includes shard contributions from all snapshots (current and completed) + handleMaximumAllowedShardCountExceeded(repositoryName, totalShardsRequiredInResponse, false); } return unmodifiableMap(snapshotsInfoMap); } @@ -492,52 +582,46 @@ private Map snapshotShards( final RepositoryData repositoryData, final SnapshotInfo snapshotInfo ) throws IOException { - final Set requestedIndexNames = Sets.newHashSet(request.indices()); String snapshotName = snapshotInfo.snapshotId().getName(); - Set indices = Sets.newHashSet(snapshotInfo.indices()); - if (requestedIndexNames.isEmpty() == false) { - Set finalIndices = indices; - List indicesNotFound = requestedIndexNames.stream() - .filter(i -> finalIndices.contains(i) == false) - .collect(Collectors.toList()); + Set indicesToProcess; + if (requestUsesIndexFilter) { + Set snapshotIndices = Sets.newHashSet(snapshotInfo.indices()); + Set indicesNotFound = requestedIndexNames.stream() + .filter(index -> snapshotIndices.contains(index) == false) + .collect(Collectors.toSet()); if (indicesNotFound.isEmpty() == false) { - handleIndexNotFound(String.join(", ", indicesNotFound), request, snapshotName, repositoryName); + boolean moreMissingIndicesPossible = indicesNotFound.size() == requestedIndexNames.size(); + handleIndexNotFound(requestedIndexNames, indicesNotFound, request, snapshotName, moreMissingIndicesPossible); } - indices = requestedIndexNames; + indicesToProcess = requestedIndexNames; + } else { + // all indices of this snapshot + indicesToProcess = Sets.newHashSet(snapshotInfo.indices()); } final Repository repository = repositoriesService.repository(repositoryName); boolean isV2Snapshot = snapshotInfo.getPinnedTimestamp() > 0; + + // for index filter-case and excludes shards from shallow v2 snapshots int totalShardsAcrossIndices = 0; final Map indexMetadataMap = new HashMap<>(); - - for (String index : indices) { + for (String index : indicesToProcess) { IndexId indexId = repositoryData.resolveIndexId(index); IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repositoryData, snapshotInfo.snapshotId(), indexId); if (indexMetadata != null) { - if (requestedIndexNames.isEmpty() == false && isV2Snapshot == false) { + if (requestUsesIndexFilter && isV2Snapshot == false) { totalShardsAcrossIndices += indexMetadata.getNumberOfShards(); } indexMetadataMap.put(indexId, indexMetadata); - } else if (requestedIndexNames.isEmpty() == false) { - handleIndexNotFound(index, request, snapshotName, repositoryName); + } else if (requestUsesIndexFilter) { + handleIndexNotFound(indicesToProcess, Collections.singleton(index), request, snapshotName, true); } } - if (totalShardsAcrossIndices > maximumAllowedShardCount && requestedIndexNames.isEmpty() == false && isV2Snapshot == false) { - String message = "[" - + repositoryName - + ":" - + String.join(", ", request.snapshots()) - + "]" - + " Total shard count [" - + totalShardsAcrossIndices - + "] across the requested indices [" - + requestedIndexNames.stream().collect(Collectors.joining(", ")) - + "] is more than the maximum allowed value of shard count [" - + maximumAllowedShardCount - + "] for snapshot status request"; - throw new CircuitBreakingException(message, CircuitBreaker.Durability.PERMANENT); + totalShardsRequiredInResponse += totalShardsAcrossIndices; + if (totalShardsRequiredInResponse > maximumAllowedShardCount && requestUsesIndexFilter && isV2Snapshot == false) { + // index-filter is allowed only for a single snapshot, which has to be this one + handleMaximumAllowedShardCountExceeded(request.repository(), totalShardsRequiredInResponse, false); } final Map shardStatus = new HashMap<>(); @@ -563,7 +647,6 @@ private Map snapshotShards( // could not be taken due to partial being set to false. shardSnapshotStatus = IndexShardSnapshotStatus.newFailed("skipped"); } else { - // TODO: to be refactored later if (isV2Snapshot) { shardSnapshotStatus = IndexShardSnapshotStatus.newDone(0, 0, 0, 0, 0, 0, null); } else { @@ -578,21 +661,55 @@ private Map snapshotShards( return unmodifiableMap(shardStatus); } - private void handleIndexNotFound(String index, SnapshotsStatusRequest request, String snapshotName, String repositoryName) { + private void handleIndexNotFound( + Set indicesToProcess, + Set indicesNotFound, + SnapshotsStatusRequest request, + String snapshotName, + boolean moreMissingIndicesPossible + ) throws IndexNotFoundException { + String indices = String.join(", ", indicesNotFound); + if (moreMissingIndicesPossible) { + indices = indices.concat(" and possibly more indices"); + } if (request.ignoreUnavailable()) { - // ignoring unavailable index + // ignoring unavailable indices logger.debug( "snapshot status request ignoring indices [{}], not found in snapshot[{}] in repository [{}]", - index, + indices, snapshotName, - repositoryName + request.repository() ); + + // remove unavailable indices from the set to be processed + indicesToProcess.removeAll(indicesNotFound); } else { - String cause = "indices [" + index + "] missing in snapshot [" + snapshotName + "] of repository [" + repositoryName + "]"; - throw new IndexNotFoundException(index, new IllegalArgumentException(cause)); + String cause = "indices [" + + indices + + "] missing in snapshot [" + + snapshotName + + "] of repository [" + + request.repository() + + "]"; + throw new IndexNotFoundException(indices, new IllegalArgumentException(cause)); } } + private void handleMaximumAllowedShardCountExceeded(String repositoryName, int totalContributingShards, boolean couldInvolveMoreShards) + throws CircuitBreakingException { + String shardCount = "[" + totalContributingShards + (couldInvolveMoreShards ? "+" : "") + "]"; + String message = "[" + + repositoryName + + "] Total shard count " + + shardCount + + " is more than the maximum allowed value of shard count [" + + maximumAllowedShardCount + + "] for snapshot status request. Try narrowing down the request by using a snapshot list or " + + "an index list for a singular snapshot."; + + throw new CircuitBreakingException(message, CircuitBreaker.Durability.PERMANENT); + } + private static SnapshotShardFailure findShardFailure(List shardFailures, ShardId shardId) { for (SnapshotShardFailure shardFailure : shardFailures) { if (shardId.getIndexName().equals(shardFailure.index()) && shardId.getId() == shardFailure.shardId()) { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java index 502be16f2fa8e..50380c506358f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java @@ -82,6 +82,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC snapshots = Strings.EMPTY_ARRAY; } String[] indices = request.paramAsStringArray("index", Strings.EMPTY_ARRAY); + if (indices.length == 1 && "_all".equalsIgnoreCase(indices[0])) { + indices = Strings.EMPTY_ARRAY; + } SnapshotsStatusRequest snapshotsStatusRequest = snapshotsStatusRequest(repository).snapshots(snapshots).indices(indices); snapshotsStatusRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", snapshotsStatusRequest.ignoreUnavailable())); From e3db7423c892a8c50f0bba9c65640d96488266d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:46:17 -0400 Subject: [PATCH 266/357] Bump com.microsoft.azure:msal4j from 1.17.1 to 1.17.2 in /plugins/repository-azure (#16406) * Bump com.microsoft.azure:msal4j in /plugins/repository-azure Bumps [com.microsoft.azure:msal4j](https://github.com/AzureAD/microsoft-authentication-library-for-java) from 1.17.1 to 1.17.2. - [Release notes](https://github.com/AzureAD/microsoft-authentication-library-for-java/releases) - [Changelog](https://github.com/AzureAD/microsoft-authentication-library-for-java/blob/dev/changelog.txt) - [Commits](https://github.com/AzureAD/microsoft-authentication-library-for-java/compare/v1.17.1...v1.17.2) --- updated-dependencies: - dependency-name: com.microsoft.azure:msal4j dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: opensearch-trigger-bot[bot] --- CHANGELOG.md | 2 +- plugins/repository-azure/build.gradle | 2 +- plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 | 1 - plugins/repository-azure/licenses/msal4j-1.17.2.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/msal4j-1.17.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b6b848f5642..dd3e26935ea6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134)) - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) -- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945)) +- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.2 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945), [#16406](https://github.com/opensearch-project/OpenSearch/pull/16406)) - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) - Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index d7eebe70ec303..3d6b3264d3f60 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -61,7 +61,7 @@ dependencies { // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' api "net.java.dev.jna:jna-platform:${versions.jna}" - api 'com.microsoft.azure:msal4j:1.17.1' + api 'com.microsoft.azure:msal4j:1.17.2' api 'com.nimbusds:oauth2-oidc-sdk:11.19.1' api 'com.nimbusds:nimbus-jose-jwt:9.41.1' api 'com.nimbusds:content-type:2.3' diff --git a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 deleted file mode 100644 index 46c14e819b630..0000000000000 --- a/plugins/repository-azure/licenses/msal4j-1.17.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4eb31a9919d9b103c548af7e37e6f9d9f6e46dbc \ No newline at end of file diff --git a/plugins/repository-azure/licenses/msal4j-1.17.2.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.17.2.jar.sha1 new file mode 100644 index 0000000000000..b5219ee17e9fa --- /dev/null +++ b/plugins/repository-azure/licenses/msal4j-1.17.2.jar.sha1 @@ -0,0 +1 @@ +a6211e3d71d0388929babaa0ff0951b30d001852 \ No newline at end of file From 322bdc42dab1d6d4fa021529057453afd5cb898e Mon Sep 17 00:00:00 2001 From: inpink <108166692+inpink@users.noreply.github.com> Date: Tue, 22 Oct 2024 03:44:26 +0900 Subject: [PATCH 267/357] [BUG] Add missing fields to resolve Strict Dynamic Mapping issue in .tasks index (#16060) (#16201) - Fixed issue where `.tasks` index failed to update due to StrictDynamicMappingException when a task was cancelled. - Added missing `cancellation_time_millis` and `resource_stats` fields to `task-index-mapping.json`. - Ensured proper task result storage by updating the mappings. - Changed the version in the meta field from 4 to 5 to reflect the updated mappings. Signed-off-by: inpink Signed-off-by: Daniel Widdis Co-authored-by: Daniel Widdis --- CHANGELOG.md | 1 + .../admin/cluster/node/tasks/TasksIT.java | 130 +++++++++++++----- .../opensearch/tasks/TaskResultsService.java | 2 +- .../opensearch/tasks/task-index-mapping.json | 9 +- .../missing-fields-task-index-mapping.json | 63 +++++++++ 5 files changed, 170 insertions(+), 35 deletions(-) create mode 100644 server/src/test/resources/org/opensearch/tasks/missing-fields-task-index-mapping.json diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3e26935ea6e..ac6117ebff4f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) - Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) +- Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) - Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) ### Security diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/TasksIT.java index c7d75108883dd..4d8c80954cd0a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/TasksIT.java @@ -58,9 +58,14 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.Streams; import org.opensearch.core.action.ActionListener; import org.opensearch.core.tasks.TaskId; +import org.opensearch.core.tasks.resourcetracker.TaskResourceStats; +import org.opensearch.core.tasks.resourcetracker.TaskResourceUsage; +import org.opensearch.core.tasks.resourcetracker.TaskThreadUsage; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.index.mapper.StrictDynamicMappingException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; @@ -73,11 +78,17 @@ import org.opensearch.transport.ReceiveTimeoutTransportException; import org.opensearch.transport.TransportService; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; @@ -103,6 +114,8 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; /** * Integration tests for task management API @@ -112,6 +125,26 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 2) public class TasksIT extends AbstractTasksIT { + protected final TaskInfo taskInfo = new TaskInfo( + new TaskId("fake", 1), + "test_type", + "test_action", + "test_description", + null, + 0L, + 1L, + false, + false, + TaskId.EMPTY_TASK_ID, + Collections.emptyMap(), + new TaskResourceStats(new HashMap<>() { + { + put("dummy-type1", new TaskResourceUsage(10, 20)); + } + }, new TaskThreadUsage(30, 40)), + 2L + ); + public void testTaskCounts() { // Run only on data nodes ListTasksResponse response = client().admin() @@ -879,46 +912,77 @@ public void testNodeNotFoundButTaskFound() throws Exception { // Save a fake task that looks like it is from a node that isn't part of the cluster CyclicBarrier b = new CyclicBarrier(2); TaskResultsService resultsService = internalCluster().getInstance(TaskResultsService.class); - resultsService.storeResult( - new TaskResult( - new TaskInfo( - new TaskId("fake", 1), - "test", - "test", - "", - null, - 0, - 0, - false, - false, - TaskId.EMPTY_TASK_ID, - Collections.emptyMap(), - null - ), - new RuntimeException("test") - ), - new ActionListener() { + resultsService.storeResult(new TaskResult(taskInfo, new RuntimeException("test")), new ActionListener() { + @Override + public void onResponse(Void response) { + try { + b.await(); + } catch (InterruptedException | BrokenBarrierException e) { + onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + }); + b.await(); + + // Now we can find it! + GetTaskResponse response = expectFinishedTask(new TaskId("fake:1")); + TaskResult taskResult = response.getTask(); + TaskInfo task = taskResult.getTask(); + + assertEquals("fake", task.getTaskId().getNodeId()); + assertEquals(1, task.getTaskId().getId()); + assertEquals("test_type", task.getType()); + assertEquals("test_action", task.getAction()); + assertEquals("test_description", task.getDescription()); + assertEquals(0L, task.getStartTime()); + assertEquals(1L, task.getRunningTimeNanos()); + assertFalse(task.isCancellable()); + assertFalse(task.isCancelled()); + assertEquals(TaskId.EMPTY_TASK_ID, task.getParentTaskId()); + assertEquals(1, task.getResourceStats().getResourceUsageInfo().size()); + assertEquals(30, task.getResourceStats().getThreadUsage().getThreadExecutions()); + assertEquals(40, task.getResourceStats().getThreadUsage().getActiveThreads()); + assertEquals(Long.valueOf(2L), task.getCancellationStartTime()); + + assertNotNull(taskResult.getError()); + assertNull(taskResult.getResponse()); + } + + public void testStoreTaskResultFailsDueToMissingIndexMappingFields() throws IOException { + // given + TaskResultsService resultsService = spy(internalCluster().getInstance(TaskResultsService.class)); + + InputStream mockInputStream = getClass().getResourceAsStream("/org/opensearch/tasks/missing-fields-task-index-mapping.json"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(mockInputStream, out); + String mockJsonString = out.toString(StandardCharsets.UTF_8.name()); + + // when & then + doReturn(mockJsonString).when(resultsService).taskResultIndexMapping(); + + CompletionException thrown = assertThrows(CompletionException.class, () -> { + CompletableFuture future = new CompletableFuture<>(); + + resultsService.storeResult(new TaskResult(taskInfo, new RuntimeException("test")), new ActionListener() { @Override public void onResponse(Void response) { - try { - b.await(); - } catch (InterruptedException | BrokenBarrierException e) { - onFailure(e); - } + future.complete(null); } @Override public void onFailure(Exception e) { - throw new RuntimeException(e); + future.completeExceptionally(e); } - } - ); - b.await(); + }); - // Now we can find it! - GetTaskResponse response = expectFinishedTask(new TaskId("fake:1")); - assertEquals("test", response.getTask().getTask().getAction()); - assertNotNull(response.getTask().getError()); - assertNull(response.getTask().getResponse()); + future.join(); + }); + + assertTrue(thrown.getCause() instanceof StrictDynamicMappingException); } } diff --git a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java index d1ee04bd5cb25..3d11bf77ae32a 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java @@ -85,7 +85,7 @@ public class TaskResultsService { public static final String TASK_RESULT_MAPPING_VERSION_META_FIELD = "version"; - public static final int TASK_RESULT_MAPPING_VERSION = 4; // must match version in task-index-mapping.json + public static final int TASK_RESULT_MAPPING_VERSION = 5; // must match version in task-index-mapping.json /** * The backoff policy to use when saving a task result fails. The total wait diff --git a/server/src/main/resources/org/opensearch/tasks/task-index-mapping.json b/server/src/main/resources/org/opensearch/tasks/task-index-mapping.json index 58b6b2d3bc873..21d418c472898 100644 --- a/server/src/main/resources/org/opensearch/tasks/task-index-mapping.json +++ b/server/src/main/resources/org/opensearch/tasks/task-index-mapping.json @@ -1,7 +1,7 @@ { "_doc" : { "_meta": { - "version": 4 + "version": 5 }, "dynamic" : "strict", "properties" : { @@ -34,6 +34,9 @@ "start_time_in_millis": { "type": "long" }, + "cancellation_time_millis": { + "type": "long" + }, "type": { "type": "keyword" }, @@ -47,6 +50,10 @@ "headers": { "type" : "object", "enabled" : false + }, + "resource_stats": { + "type" : "object", + "enabled" : false } } }, diff --git a/server/src/test/resources/org/opensearch/tasks/missing-fields-task-index-mapping.json b/server/src/test/resources/org/opensearch/tasks/missing-fields-task-index-mapping.json new file mode 100644 index 0000000000000..2e59bbc4803bf --- /dev/null +++ b/server/src/test/resources/org/opensearch/tasks/missing-fields-task-index-mapping.json @@ -0,0 +1,63 @@ +{ + "_doc" : { + "_meta": { + "version": 5 + }, + "dynamic" : "strict", + "properties" : { + "completed": { + "type": "boolean" + }, + "task" : { + "properties": { + "action": { + "type": "keyword" + }, + "cancellable": { + "type": "boolean" + }, + "cancelled": { + "type": "boolean" + }, + "id": { + "type": "long" + }, + "parent_task_id": { + "type": "keyword" + }, + "node": { + "type": "keyword" + }, + "running_time_in_nanos": { + "type": "long" + }, + "start_time_in_millis": { + "type": "long" + }, + "type": { + "type": "keyword" + }, + "status": { + "type" : "object", + "enabled" : false + }, + "description": { + "type": "text" + }, + "headers": { + "type" : "object", + "enabled" : false + } + } + }, + "response" : { + "type" : "object", + "enabled" : false + }, + "error" : { + "type" : "object", + "enabled" : false + } + } + } +} From 456ca97f077929ba51d53dc8578b9b770e3a2417 Mon Sep 17 00:00:00 2001 From: Sandesh Kumar Date: Mon, 21 Oct 2024 12:34:58 -0700 Subject: [PATCH 268/357] [Star Tree] [Search] Support for metric aggregations with/without term query (#15289) --------- Signed-off-by: Sandesh Kumar --- CHANGELOG.md | 1 + .../datacube/DateDimension.java | 5 + .../compositeindex/datacube/Dimension.java | 3 + .../datacube/NumericDimension.java | 6 + .../datacube/ReadDimension.java | 6 + .../startree/utils/StarTreeQueryHelper.java | 248 ++++++++++++++ .../SortedNumericStarTreeValuesIterator.java | 8 + .../org/opensearch/search/SearchService.java | 26 +- .../aggregations/AggregatorFactories.java | 4 + .../aggregations/AggregatorFactory.java | 4 + .../aggregations/metrics/AvgAggregator.java | 74 ++++ .../metrics/AvgAggregatorFactory.java | 9 +- .../aggregations/metrics/MaxAggregator.java | 34 ++ .../metrics/MaxAggregatorFactory.java | 9 +- .../metrics/MetricAggregatorFactory.java | 37 ++ .../aggregations/metrics/MinAggregator.java | 34 +- .../metrics/MinAggregatorFactory.java | 9 +- .../aggregations/metrics/SumAggregator.java | 30 ++ .../metrics/SumAggregatorFactory.java | 9 +- .../metrics/ValueCountAggregator.java | 26 +- .../metrics/ValueCountAggregatorFactory.java | 9 +- .../aggregations/support/ValuesSource.java | 4 + .../ValuesSourceAggregatorFactory.java | 4 + .../search/internal/SearchContext.java | 11 +- .../search/startree/StarTreeFilter.java | 228 +++++++++++++ .../search/startree/StarTreeQueryContext.java | 79 +++++ .../search/startree/package-info.java | 10 + .../StarTreeDocValuesFormatTests.java | 11 +- .../search/SearchServiceStarTreeTests.java | 160 +++++++++ .../startree/MetricAggregatorTests.java | 317 +++++++++++++++++ .../startree/StarTreeFilterTests.java | 319 ++++++++++++++++++ .../aggregations/AggregatorTestCase.java | 142 ++++++++ 32 files changed, 1853 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java create mode 100644 server/src/main/java/org/opensearch/search/aggregations/metrics/MetricAggregatorFactory.java create mode 100644 server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java create mode 100644 server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java create mode 100644 server/src/main/java/org/opensearch/search/startree/package-info.java create mode 100644 server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java create mode 100644 server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java create mode 100644 server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ac6117ebff4f6..76b65a6cd70dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) - Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387) +- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index ee6d5b4680c73..8feb9ccd27dbd 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube; +import org.apache.lucene.index.DocValuesType; import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.time.DateUtils; @@ -169,4 +170,8 @@ public int compare(DateTimeUnitRounding unit1, DateTimeUnitRounding unit2) { public static List getSortedDateTimeUnits(List dateTimeUnits) { return dateTimeUnits.stream().sorted(new DateTimeUnitComparator()).collect(Collectors.toList()); } + + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_NUMERIC; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index cfa8d3a2a8164..3d71b38881693 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube; +import org.apache.lucene.index.DocValuesType; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; @@ -42,4 +43,6 @@ public interface Dimension extends ToXContent { * Returns the list of dimension fields that represent the dimension */ List getSubDimensionNames(); + + DocValuesType getDocValuesType(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index acc14f5f05c68..f1d1b15337f4a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube; +import org.apache.lucene.index.DocValuesType; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.CompositeDataCubeFieldType; @@ -71,4 +72,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } + + @Override + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_NUMERIC; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index be3667f10b6da..0e2ec086abc0a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube; +import org.apache.lucene.index.DocValuesType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.CompositeDataCubeFieldType; @@ -69,4 +70,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } + + @Override + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_NUMERIC; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java new file mode 100644 index 0000000000000..e538be5d5bece --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java @@ -0,0 +1,248 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.LeafBucketCollectorBase; +import org.opensearch.search.aggregations.metrics.MetricAggregatorFactory; +import org.opensearch.search.aggregations.support.ValuesSource; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeFilter; +import org.opensearch.search.startree.StarTreeQueryContext; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Helper class for building star-tree query + * + * @opensearch.internal + * @opensearch.experimental + */ +public class StarTreeQueryHelper { + + /** + * Checks if the search context can be supported by star-tree + */ + public static boolean isStarTreeSupported(SearchContext context) { + return context.aggregations() != null && context.mapperService().isCompositeIndexPresent() && context.parsedPostFilter() == null; + } + + /** + * Gets StarTreeQueryContext from the search context and source builder. + * Returns null if the query and aggregation cannot be supported. + */ + public static StarTreeQueryContext getStarTreeQueryContext(SearchContext context, SearchSourceBuilder source) throws IOException { + // Current implementation assumes only single star-tree is supported + CompositeDataCubeFieldType compositeMappedFieldType = (CompositeDataCubeFieldType) context.mapperService() + .getCompositeFieldTypes() + .iterator() + .next(); + CompositeIndexFieldInfo starTree = new CompositeIndexFieldInfo( + compositeMappedFieldType.name(), + compositeMappedFieldType.getCompositeIndexType() + ); + + for (AggregatorFactory aggregatorFactory : context.aggregations().factories().getFactories()) { + MetricStat metricStat = validateStarTreeMetricSupport(compositeMappedFieldType, aggregatorFactory); + if (metricStat == null) { + return null; + } + } + + // need to cache star tree values only for multiple aggregations + boolean cacheStarTreeValues = context.aggregations().factories().getFactories().length > 1; + int cacheSize = cacheStarTreeValues ? context.indexShard().segments(false).size() : -1; + + return StarTreeQueryHelper.tryCreateStarTreeQueryContext(starTree, compositeMappedFieldType, source.query(), cacheSize); + } + + /** + * Uses query builder and composite index info to form star-tree query context + */ + private static StarTreeQueryContext tryCreateStarTreeQueryContext( + CompositeIndexFieldInfo compositeIndexFieldInfo, + CompositeDataCubeFieldType compositeFieldType, + QueryBuilder queryBuilder, + int cacheStarTreeValuesSize + ) { + Map queryMap; + if (queryBuilder == null || queryBuilder instanceof MatchAllQueryBuilder) { + queryMap = null; + } else if (queryBuilder instanceof TermQueryBuilder) { + // TODO: Add support for keyword fields + if (compositeFieldType.getDimensions().stream().anyMatch(d -> d.getDocValuesType() != DocValuesType.SORTED_NUMERIC)) { + // return null for non-numeric fields + return null; + } + + List supportedDimensions = compositeFieldType.getDimensions() + .stream() + .map(Dimension::getField) + .collect(Collectors.toList()); + queryMap = getStarTreePredicates(queryBuilder, supportedDimensions); + if (queryMap == null) { + return null; + } + } else { + return null; + } + return new StarTreeQueryContext(compositeIndexFieldInfo, queryMap, cacheStarTreeValuesSize); + } + + /** + * Parse query body to star-tree predicates + * @param queryBuilder to match star-tree supported query shape + * @return predicates to match + */ + private static Map getStarTreePredicates(QueryBuilder queryBuilder, List supportedDimensions) { + TermQueryBuilder tq = (TermQueryBuilder) queryBuilder; + String field = tq.fieldName(); + if (!supportedDimensions.contains(field)) { + return null; + } + long inputQueryVal = Long.parseLong(tq.value().toString()); + + // Create a map with the field and the value + Map predicateMap = new HashMap<>(); + predicateMap.put(field, inputQueryVal); + return predicateMap; + } + + private static MetricStat validateStarTreeMetricSupport( + CompositeDataCubeFieldType compositeIndexFieldInfo, + AggregatorFactory aggregatorFactory + ) { + if (aggregatorFactory instanceof MetricAggregatorFactory && aggregatorFactory.getSubFactories().getFactories().length == 0) { + String field; + Map> supportedMetrics = compositeIndexFieldInfo.getMetrics() + .stream() + .collect(Collectors.toMap(Metric::getField, Metric::getMetrics)); + + MetricStat metricStat = ((MetricAggregatorFactory) aggregatorFactory).getMetricStat(); + field = ((MetricAggregatorFactory) aggregatorFactory).getField(); + + if (supportedMetrics.containsKey(field) && supportedMetrics.get(field).contains(metricStat)) { + return metricStat; + } + } + return null; + } + + public static CompositeIndexFieldInfo getSupportedStarTree(SearchContext context) { + StarTreeQueryContext starTreeQueryContext = context.getStarTreeQueryContext(); + return (starTreeQueryContext != null) ? starTreeQueryContext.getStarTree() : null; + } + + public static StarTreeValues getStarTreeValues(LeafReaderContext context, CompositeIndexFieldInfo starTree) throws IOException { + SegmentReader reader = Lucene.segmentReader(context.reader()); + if (!(reader.getDocValuesReader() instanceof CompositeIndexReader)) { + return null; + } + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + return (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(starTree); + } + + /** + * Get the star-tree leaf collector + * This collector computes the aggregation prematurely and invokes an early termination collector + */ + public static LeafBucketCollector getStarTreeLeafCollector( + SearchContext context, + ValuesSource.Numeric valuesSource, + LeafReaderContext ctx, + LeafBucketCollector sub, + CompositeIndexFieldInfo starTree, + String metric, + Consumer valueConsumer, + Runnable finalConsumer + ) throws IOException { + StarTreeValues starTreeValues = getStarTreeValues(ctx, starTree); + assert starTreeValues != null; + String fieldName = ((ValuesSource.Numeric.FieldData) valuesSource).getIndexFieldName(); + String metricName = StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues(starTree.getField(), fieldName, metric); + + assert starTreeValues != null; + SortedNumericStarTreeValuesIterator valuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues.getMetricValuesIterator( + metricName + ); + // Obtain a FixedBitSet of matched star tree document IDs + FixedBitSet filteredValues = getStarTreeFilteredValues(context, ctx, starTreeValues); + assert filteredValues != null; + + int numBits = filteredValues.length(); // Get the number of the filtered values (matching docs) + if (numBits > 0) { + // Iterate over the filtered values + for (int bit = filteredValues.nextSetBit(0); bit != DocIdSetIterator.NO_MORE_DOCS; bit = (bit + 1 < numBits) + ? filteredValues.nextSetBit(bit + 1) + : DocIdSetIterator.NO_MORE_DOCS) { + // Advance to the entryId in the valuesIterator + if (valuesIterator.advanceExact(bit) == false) { + continue; // Skip if no more entries + } + + // Iterate over the values for the current entryId + for (int i = 0, count = valuesIterator.entryValueCount(); i < count; i++) { + long value = valuesIterator.nextValue(); + valueConsumer.accept(value); // Apply the consumer operation (e.g., max, sum) + } + } + } + + // Call the final consumer after processing all entries + finalConsumer.run(); + + // Return a LeafBucketCollector that terminates collection + return new LeafBucketCollectorBase(sub, valuesSource.doubleValues(ctx)) { + @Override + public void collect(int doc, long bucket) { + throw new CollectionTerminatedException(); + } + }; + } + + /** + * Get the filtered values for the star-tree query + * Cache the results in case of multiple aggregations (if cache is initialized) + * @return FixedBitSet of matched document IDs + */ + public static FixedBitSet getStarTreeFilteredValues(SearchContext context, LeafReaderContext ctx, StarTreeValues starTreeValues) + throws IOException { + FixedBitSet result = context.getStarTreeQueryContext().getStarTreeValues(ctx); + if (result == null) { + result = StarTreeFilter.getStarTreeResult(starTreeValues, context.getStarTreeQueryContext().getQueryMap()); + context.getStarTreeQueryContext().setStarTreeValues(ctx, result); + } + return result; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java index 27afdf1479b4e..4b4bfa6a915eb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -29,4 +29,12 @@ public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { public long nextValue() throws IOException { return ((SortedNumericDocValues) docIdSetIterator).nextValue(); } + + public int entryValueCount() throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).docValueCount(); + } + + public boolean advanceExact(int target) throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).advanceExact(target); + } } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index c2f8b17fcf166..e892a2f1a7620 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -77,6 +77,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.DerivedFieldResolver; import org.opensearch.index.mapper.DerivedFieldResolverFactory; @@ -137,6 +138,7 @@ import org.opensearch.search.sort.SortAndFormats; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortOrder; +import org.opensearch.search.startree.StarTreeQueryContext; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.completion.CompletionSuggestion; import org.opensearch.tasks.TaskResourceTrackingService; @@ -164,6 +166,7 @@ import static org.opensearch.common.unit.TimeValue.timeValueHours; import static org.opensearch.common.unit.TimeValue.timeValueMillis; import static org.opensearch.common.unit.TimeValue.timeValueMinutes; +import static org.opensearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** * The main search service @@ -1358,6 +1361,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.evaluateRequestShouldUseConcurrentSearch(); return; } + SearchShardTarget shardTarget = context.shardTarget(); QueryShardContext queryShardContext = context.getQueryShardContext(); context.from(source.from()); @@ -1371,7 +1375,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc InnerHitContextBuilder.extractInnerHits(source.postFilter(), innerHitBuilders); context.parsedPostFilter(queryShardContext.toQuery(source.postFilter())); } - if (innerHitBuilders.size() > 0) { + if (!innerHitBuilders.isEmpty()) { for (Map.Entry entry : innerHitBuilders.entrySet()) { try { entry.getValue().build(context, context.innerHits()); @@ -1383,9 +1387,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc if (source.sorts() != null) { try { Optional optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext()); - if (optionalSort.isPresent()) { - context.sort(optionalSort.get()); - } + optionalSort.ifPresent(context::sort); } catch (IOException e) { throw new SearchException(shardTarget, "failed to create sort elements", e); } @@ -1540,6 +1542,20 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc if (source.profile()) { context.setProfilers(new Profilers(context.searcher(), context.shouldUseConcurrentSearch())); } + + if (this.indicesService.getCompositeIndexSettings() != null + && this.indicesService.getCompositeIndexSettings().isStarTreeIndexCreationEnabled() + && StarTreeQueryHelper.isStarTreeSupported(context)) { + try { + StarTreeQueryContext starTreeQueryContext = StarTreeQueryHelper.getStarTreeQueryContext(context, source); + if (starTreeQueryContext != null) { + context.starTreeQueryContext(starTreeQueryContext); + logger.debug("can use star tree"); + } else { + logger.debug("cannot use star tree"); + } + } catch (IOException ignored) {} + } } /** @@ -1699,7 +1715,7 @@ public static boolean canMatchSearchAfter( && minMax != null && primarySortField != null && primarySortField.missing() == null - && Objects.equals(trackTotalHitsUpto, SearchContext.TRACK_TOTAL_HITS_DISABLED)) { + && Objects.equals(trackTotalHitsUpto, TRACK_TOTAL_HITS_DISABLED)) { final Object searchAfterPrimary = searchAfter.fields[0]; if (primarySortField.order() == SortOrder.DESC) { if (minMax.compareMin(searchAfterPrimary) > 0) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java index eeb0c606694b0..720a24da1d9d4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactories.java @@ -661,4 +661,8 @@ public PipelineTree buildPipelineTree() { return new PipelineTree(subTrees, aggregators); } } + + public AggregatorFactory[] getFactories() { + return factories; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactory.java index 6cc3a78fb1e36..86fbb46a9ad3c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorFactory.java @@ -127,4 +127,8 @@ protected boolean supportsConcurrentSegmentSearch() { public boolean evaluateChildFactories() { return factories.allFactoriesSupportConcurrentSearch(); } + + public AggregatorFactories getSubFactories() { + return factories; + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java index e58466b56df2a..2970c5ca851e7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java @@ -32,11 +32,21 @@ package org.opensearch.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.NumericUtils; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.DoubleArray; import org.opensearch.common.util.LongArray; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregator; @@ -50,6 +60,9 @@ import java.io.IOException; import java.util.Map; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getStarTreeFilteredValues; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; + /** * Aggregate all docs into an average * @@ -93,6 +106,14 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + if (supportedStarTree != null) { + return getStarTreeLeafCollector(ctx, sub, supportedStarTree); + } + return getDefaultLeafCollector(ctx, sub); + } + + private LeafBucketCollector getDefaultLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); final CompensatedSum kahanSummation = new CompensatedSum(0, 0); @@ -126,6 +147,59 @@ public void collect(int doc, long bucket) throws IOException { }; } + public LeafBucketCollector getStarTreeLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub, CompositeIndexFieldInfo starTree) + throws IOException { + StarTreeValues starTreeValues = StarTreeQueryHelper.getStarTreeValues(ctx, starTree); + assert starTreeValues != null; + + String fieldName = ((ValuesSource.Numeric.FieldData) valuesSource).getIndexFieldName(); + String sumMetricName = StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTree.getField(), + fieldName, + MetricStat.SUM.getTypeName() + ); + String countMetricName = StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTree.getField(), + fieldName, + MetricStat.VALUE_COUNT.getTypeName() + ); + + final CompensatedSum kahanSummation = new CompensatedSum(sums.get(0), 0); + SortedNumericStarTreeValuesIterator sumValuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues + .getMetricValuesIterator(sumMetricName); + SortedNumericStarTreeValuesIterator countValueIterator = (SortedNumericStarTreeValuesIterator) starTreeValues + .getMetricValuesIterator(countMetricName); + FixedBitSet matchedDocIds = getStarTreeFilteredValues(context, ctx, starTreeValues); + assert matchedDocIds != null; + + int numBits = matchedDocIds.length(); // Get the length of the FixedBitSet + if (numBits > 0) { + // Iterate over the FixedBitSet + for (int bit = matchedDocIds.nextSetBit(0); bit != DocIdSetIterator.NO_MORE_DOCS; bit = bit + 1 < numBits + ? matchedDocIds.nextSetBit(bit + 1) + : DocIdSetIterator.NO_MORE_DOCS) { + // Advance to the bit (entryId) in the valuesIterator + if ((sumValuesIterator.advanceExact(bit) && countValueIterator.advanceExact(bit)) == false) { + continue; // Skip if no more entries + } + + // Iterate over the values for the current entryId + for (int i = 0; i < sumValuesIterator.entryValueCount(); i++) { + kahanSummation.add(NumericUtils.sortableLongToDouble(sumValuesIterator.nextValue())); + counts.increment(0, countValueIterator.nextValue()); // Apply the consumer operation (e.g., max, sum) + } + } + } + + sums.set(0, kahanSummation.value()); + return new LeafBucketCollectorBase(sub, valuesSource.doubleValues(ctx)) { + @Override + public void collect(int doc, long bucket) { + throw new CollectionTerminatedException(); + } + }; + } + @Override public double metric(long owningBucketOrd) { if (valuesSource == null || owningBucketOrd >= sums.size()) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java index 0a09fae1eaebe..57389f19b4577 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregatorFactory.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.internal.SearchContext; @@ -52,7 +52,7 @@ * * @opensearch.internal */ -class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { +class AvgAggregatorFactory extends MetricAggregatorFactory { AvgAggregatorFactory( String name, @@ -65,6 +65,11 @@ class AvgAggregatorFactory extends ValuesSourceAggregatorFactory { super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); } + @Override + public MetricStat getMetricStat() { + return MetricStat.AVG; + } + static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( AvgAggregationBuilder.REGISTRY_KEY, diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java index 8108b8a726856..257109bca54bb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java @@ -37,9 +37,13 @@ import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.NumericUtils; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.DoubleArray; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; @@ -55,8 +59,11 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; + /** * Aggregate all docs into a max value * @@ -120,6 +127,16 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc throw new CollectionTerminatedException(); } } + + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + if (supportedStarTree != null) { + return getStarTreeCollector(ctx, sub, supportedStarTree); + } + return getDefaultLeafCollector(ctx, sub); + } + + private LeafBucketCollector getDefaultLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues allValues = valuesSource.doubleValues(ctx); final NumericDoubleValues values = MultiValueMode.MAX.select(allValues); @@ -143,6 +160,23 @@ public void collect(int doc, long bucket) throws IOException { }; } + public LeafBucketCollector getStarTreeCollector(LeafReaderContext ctx, LeafBucketCollector sub, CompositeIndexFieldInfo starTree) + throws IOException { + AtomicReference max = new AtomicReference<>(maxes.get(0)); + return StarTreeQueryHelper.getStarTreeLeafCollector( + context, + valuesSource, + ctx, + sub, + starTree, + MetricStat.MAX.getTypeName(), + value -> { + max.set(Math.max(max.get(), (NumericUtils.sortableLongToDouble(value)))); + }, + () -> maxes.set(0, max.get()) + ); + } + @Override public double metric(long owningBucketOrd) { if (valuesSource == null || owningBucketOrd >= maxes.size()) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java index 4fe936c8b7797..c0ee471c87f29 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregatorFactory.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.internal.SearchContext; @@ -52,7 +52,7 @@ * * @opensearch.internal */ -class MaxAggregatorFactory extends ValuesSourceAggregatorFactory { +class MaxAggregatorFactory extends MetricAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( @@ -74,6 +74,11 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) { super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); } + @Override + public MetricStat getMetricStat() { + return MetricStat.MAX; + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { return new MaxAggregator(name, config, searchContext, parent, metadata); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MetricAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MetricAggregatorFactory.java new file mode 100644 index 0000000000000..0ac630cf051d3 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MetricAggregatorFactory.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.metrics; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.aggregations.AggregatorFactories; +import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.opensearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.Map; + +/** + * Extending ValuesSourceAggregatorFactory for aggregation factories supported by star-tree implementation + */ +public abstract class MetricAggregatorFactory extends ValuesSourceAggregatorFactory { + public MetricAggregatorFactory( + String name, + ValuesSourceConfig config, + QueryShardContext queryShardContext, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata + ) throws IOException { + super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); + } + + public abstract MetricStat getMetricStat(); +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java index 946057e42ac88..a9f20bdeb5fd5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java @@ -37,9 +37,13 @@ import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.NumericUtils; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.DoubleArray; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; @@ -54,8 +58,11 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; + /** * Aggregate all docs into a min value * @@ -119,6 +126,15 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc throw new CollectionTerminatedException(); } } + + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + if (supportedStarTree != null) { + return getStarTreeCollector(ctx, sub, supportedStarTree); + } + return getDefaultLeafCollector(ctx, sub); + } + + private LeafBucketCollector getDefaultLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues allValues = valuesSource.doubleValues(ctx); final NumericDoubleValues values = MultiValueMode.MIN.select(allValues); @@ -138,10 +154,26 @@ public void collect(int doc, long bucket) throws IOException { mins.set(bucket, min); } } - }; } + public LeafBucketCollector getStarTreeCollector(LeafReaderContext ctx, LeafBucketCollector sub, CompositeIndexFieldInfo starTree) + throws IOException { + AtomicReference min = new AtomicReference<>(mins.get(0)); + return StarTreeQueryHelper.getStarTreeLeafCollector( + context, + valuesSource, + ctx, + sub, + starTree, + MetricStat.MIN.getTypeName(), + value -> { + min.set(Math.min(min.get(), (NumericUtils.sortableLongToDouble(value)))); + }, + () -> mins.set(0, min.get()) + ); + } + @Override public double metric(long owningBucketOrd) { if (valuesSource == null || owningBucketOrd >= mins.size()) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java index 58fbe5edefd12..44c0d9d7d11eb 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregatorFactory.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.internal.SearchContext; @@ -52,7 +52,7 @@ * * @opensearch.internal */ -class MinAggregatorFactory extends ValuesSourceAggregatorFactory { +class MinAggregatorFactory extends MetricAggregatorFactory { static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( @@ -74,6 +74,11 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) { super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); } + @Override + public MetricStat getMetricStat() { + return MetricStat.MIN; + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { return new MinAggregator(name, config, searchContext, parent, metadata); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java index 4b8e882cd69bc..3d237a94c5699 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java @@ -33,9 +33,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.util.NumericUtils; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.DoubleArray; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregator; @@ -49,6 +53,8 @@ import java.io.IOException; import java.util.Map; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; + /** * Aggregate all docs into a single sum value * @@ -89,6 +95,15 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } + + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + if (supportedStarTree != null) { + return getStarTreeCollector(ctx, sub, supportedStarTree); + } + return getDefaultLeafCollector(ctx, sub); + } + + private LeafBucketCollector getDefaultLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); final CompensatedSum kahanSummation = new CompensatedSum(0, 0); @@ -118,6 +133,21 @@ public void collect(int doc, long bucket) throws IOException { }; } + public LeafBucketCollector getStarTreeCollector(LeafReaderContext ctx, LeafBucketCollector sub, CompositeIndexFieldInfo starTree) + throws IOException { + final CompensatedSum kahanSummation = new CompensatedSum(sums.get(0), 0); + return StarTreeQueryHelper.getStarTreeLeafCollector( + context, + valuesSource, + ctx, + sub, + starTree, + MetricStat.SUM.getTypeName(), + value -> kahanSummation.add(NumericUtils.sortableLongToDouble(value)), + () -> sums.set(0, kahanSummation.value()) + ); + } + @Override public double metric(long owningBucketOrd) { if (valuesSource == null || owningBucketOrd >= sums.size()) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java index ef9b93920ba18..e2e25a8c25a87 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregatorFactory.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.internal.SearchContext; @@ -52,7 +52,7 @@ * * @opensearch.internal */ -class SumAggregatorFactory extends ValuesSourceAggregatorFactory { +class SumAggregatorFactory extends MetricAggregatorFactory { SumAggregatorFactory( String name, @@ -65,6 +65,11 @@ class SumAggregatorFactory extends ValuesSourceAggregatorFactory { super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); } + @Override + public MetricStat getMetricStat() { + return MetricStat.SUM; + } + static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register( SumAggregationBuilder.REGISTRY_KEY, diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java index 6f9be06231819..a156ec49983fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java @@ -37,6 +37,9 @@ import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.LongArray; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.MultiGeoPointValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.search.aggregations.Aggregator; @@ -50,6 +53,8 @@ import java.io.IOException; import java.util.Map; +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; + /** * A field data based aggregator that counts the number of values a specific field has within the aggregation context. *

    @@ -88,6 +93,12 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc final BigArrays bigArrays = context.bigArrays(); if (valuesSource instanceof ValuesSource.Numeric) { + + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + if (supportedStarTree != null) { + return getStarTreeCollector(ctx, sub, supportedStarTree); + } + final SortedNumericDocValues values = ((ValuesSource.Numeric) valuesSource).longValues(ctx); return new LeafBucketCollectorBase(sub, values) { @@ -124,10 +135,23 @@ public void collect(int doc, long bucket) throws IOException { counts.increment(bucket, values.docValueCount()); } } - }; } + public LeafBucketCollector getStarTreeCollector(LeafReaderContext ctx, LeafBucketCollector sub, CompositeIndexFieldInfo starTree) + throws IOException { + return StarTreeQueryHelper.getStarTreeLeafCollector( + context, + (ValuesSource.Numeric) valuesSource, + ctx, + sub, + starTree, + MetricStat.VALUE_COUNT.getTypeName(), + value -> counts.increment(0, value), + () -> {} + ); + } + @Override public double metric(long owningBucketOrd) { return (valuesSource == null || owningBucketOrd >= counts.size()) ? 0 : counts.get(owningBucketOrd); diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java index 4a04dd2e0a932..0c82279484461 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregatorFactory.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; +import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.internal.SearchContext; @@ -51,7 +51,7 @@ * * @opensearch.internal */ -class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory { +class ValueCountAggregatorFactory extends MetricAggregatorFactory { public static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register(ValueCountAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.ALL_CORE, ValueCountAggregator::new, true); @@ -68,6 +68,11 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { super(name, config, queryShardContext, parent, subFactoriesBuilder, metadata); } + @Override + public MetricStat getMetricStat() { + return MetricStat.VALUE_COUNT; + } + @Override protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { return new ValueCountAggregator(name, config, searchContext, parent, metadata); diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java index 1f4dd429e094e..5732d545cb2d2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSource.java @@ -625,6 +625,10 @@ public SortedNumericDocValues longValues(LeafReaderContext context) { public SortedNumericDoubleValues doubleValues(LeafReaderContext context) { return indexFieldData.load(context).getDoubleValues(); } + + public String getIndexFieldName() { + return indexFieldData.getFieldName(); + } } /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregatorFactory.java index 69a4a5d8b6703..d862b2c2784de 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregatorFactory.java @@ -102,4 +102,8 @@ protected abstract Aggregator doCreateInternal( public String getStatsSubtype() { return config.valueSourceType().typeName(); } + + public String getField() { + return config.fieldContext().field(); + } } diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index 5357206e8c117..b7ea06d2989e5 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -76,6 +76,7 @@ import org.opensearch.search.query.ReduceableSearchResult; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.sort.SortAndFormats; +import org.opensearch.search.startree.StarTreeQueryContext; import org.opensearch.search.suggest.SuggestionSearchContext; import java.util.Collection; @@ -124,8 +125,8 @@ public List toInternalAggregations(Collection co private final List releasables = new CopyOnWriteArrayList<>(); private final AtomicBoolean closed = new AtomicBoolean(false); private InnerHitsContext innerHitsContext; - private volatile boolean searchTimedOut; + private StarTreeQueryContext starTreeQueryContext; protected SearchContext() {} @@ -531,4 +532,12 @@ public boolean keywordIndexOrDocValuesEnabled() { return false; } + public SearchContext starTreeQueryContext(StarTreeQueryContext starTreeQueryContext) { + this.starTreeQueryContext = starTreeQueryContext; + return this; + } + + public StarTreeQueryContext getStarTreeQueryContext() { + return this.starTreeQueryContext; + } } diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java new file mode 100644 index 0000000000000..f7fa210691678 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java @@ -0,0 +1,228 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +/** + * Filter operator for star tree data structure. + * + * @opensearch.experimental + * @opensearch.internal + */ +public class StarTreeFilter { + private static final Logger logger = LogManager.getLogger(StarTreeFilter.class); + + /** + * First go over the star tree and try to match as many dimensions as possible + * For the remaining columns, use star-tree doc values to match them + */ + public static FixedBitSet getStarTreeResult(StarTreeValues starTreeValues, Map predicateEvaluators) throws IOException { + Map queryMap = predicateEvaluators != null ? predicateEvaluators : Collections.emptyMap(); + StarTreeResult starTreeResult = traverseStarTree(starTreeValues, queryMap); + + // Initialize FixedBitSet with size maxMatchedDoc + 1 + FixedBitSet bitSet = new FixedBitSet(starTreeResult.maxMatchedDoc + 1); + SortedNumericStarTreeValuesIterator starTreeValuesIterator = new SortedNumericStarTreeValuesIterator( + starTreeResult.matchedDocIds.build().iterator() + ); + + // No matches, return an empty FixedBitSet + if (starTreeResult.maxMatchedDoc == -1) { + return bitSet; + } + + // Set bits in FixedBitSet for initially matched documents + while (starTreeValuesIterator.nextEntry() != NO_MORE_DOCS) { + bitSet.set(starTreeValuesIterator.entryId()); + } + + // Temporary FixedBitSet reused for filtering + FixedBitSet tempBitSet = new FixedBitSet(starTreeResult.maxMatchedDoc + 1); + + // Process remaining predicate columns to further filter the results + for (String remainingPredicateColumn : starTreeResult.remainingPredicateColumns) { + logger.debug("remainingPredicateColumn : {}, maxMatchedDoc : {} ", remainingPredicateColumn, starTreeResult.maxMatchedDoc); + + SortedNumericStarTreeValuesIterator ndv = (SortedNumericStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator( + remainingPredicateColumn + ); + + long queryValue = queryMap.get(remainingPredicateColumn); // Get the query value directly + + // Clear the temporary bit set before reuse + tempBitSet.clear(0, starTreeResult.maxMatchedDoc + 1); + + if (bitSet.length() > 0) { + // Iterate over the current set of matched document IDs + for (int entryId = bitSet.nextSetBit(0); entryId != DocIdSetIterator.NO_MORE_DOCS; entryId = (entryId + 1 < bitSet.length()) + ? bitSet.nextSetBit(entryId + 1) + : DocIdSetIterator.NO_MORE_DOCS) { + if (ndv.advance(entryId) != StarTreeValuesIterator.NO_MORE_ENTRIES) { + final int valuesCount = ndv.entryValueCount(); + for (int i = 0; i < valuesCount; i++) { + long value = ndv.nextValue(); + // Compare the value with the query value + if (value == queryValue) { + tempBitSet.set(entryId); // Set bit for the matching entryId + break; // No need to check other values for this entryId + } + } + } + } + } + + // Perform intersection of the current matches with the temp results for this predicate + bitSet.and(tempBitSet); + } + + return bitSet; // Return the final FixedBitSet with all matches + } + + /** + * Helper method to traverse the star tree, get matching documents and keep track of all the + * predicate dimensions that are not matched. + */ + private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Map queryMap) throws IOException { + DocIdSetBuilder docsWithField = new DocIdSetBuilder(starTreeValues.getStarTreeDocumentCount()); + DocIdSetBuilder.BulkAdder adder; + Set globalRemainingPredicateColumns = null; + StarTreeNode starTree = starTreeValues.getRoot(); + List dimensionNames = starTreeValues.getStarTreeField() + .getDimensionsOrder() + .stream() + .map(Dimension::getField) + .collect(Collectors.toList()); + boolean foundLeafNode = starTree.isLeaf(); + assert foundLeafNode == false; // root node is never leaf + Queue queue = new ArrayDeque<>(); + queue.add(starTree); + int currentDimensionId = -1; + Set remainingPredicateColumns = new HashSet<>(queryMap.keySet()); + int matchedDocsCountInStarTree = 0; + int maxDocNum = -1; + StarTreeNode starTreeNode; + List docIds = new ArrayList<>(); + + while ((starTreeNode = queue.poll()) != null) { + int dimensionId = starTreeNode.getDimensionId(); + if (dimensionId > currentDimensionId) { + String dimension = dimensionNames.get(dimensionId); + remainingPredicateColumns.remove(dimension); + if (foundLeafNode && globalRemainingPredicateColumns == null) { + globalRemainingPredicateColumns = new HashSet<>(remainingPredicateColumns); + } + currentDimensionId = dimensionId; + } + + if (remainingPredicateColumns.isEmpty()) { + int docId = starTreeNode.getAggregatedDocId(); + docIds.add(docId); + matchedDocsCountInStarTree++; + maxDocNum = Math.max(docId, maxDocNum); + continue; + } + + if (starTreeNode.isLeaf()) { + for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { + docIds.add((int) i); + matchedDocsCountInStarTree++; + maxDocNum = Math.max((int) i, maxDocNum); + } + continue; + } + + String childDimension = dimensionNames.get(dimensionId + 1); + StarTreeNode starNode = null; + if (globalRemainingPredicateColumns == null || !globalRemainingPredicateColumns.contains(childDimension)) { + starNode = starTreeNode.getChildStarNode(); + } + + if (remainingPredicateColumns.contains(childDimension)) { + long queryValue = queryMap.get(childDimension); // Get the query value directly from the map + StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue); + if (matchingChild != null) { + queue.add(matchingChild); + foundLeafNode |= matchingChild.isLeaf(); + } + } else { + if (starNode != null) { + queue.add(starNode); + foundLeafNode |= starNode.isLeaf(); + } else { + Iterator childrenIterator = starTreeNode.getChildrenIterator(); + while (childrenIterator.hasNext()) { + StarTreeNode childNode = childrenIterator.next(); + if (childNode.getStarTreeNodeType() != StarTreeNodeType.STAR.getValue()) { + queue.add(childNode); + foundLeafNode |= childNode.isLeaf(); + } + } + } + } + } + + adder = docsWithField.grow(docIds.size()); + for (int id : docIds) { + adder.add(id); + } + return new StarTreeResult( + docsWithField, + globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(), + matchedDocsCountInStarTree, + maxDocNum + ); + } + + /** + * Helper class to wrap the result from traversing the star tree. + * */ + private static class StarTreeResult { + public final DocIdSetBuilder matchedDocIds; + public final Set remainingPredicateColumns; + public final int numOfMatchedDocs; + public final int maxMatchedDoc; + + public StarTreeResult( + DocIdSetBuilder matchedDocIds, + Set remainingPredicateColumns, + int numOfMatchedDocs, + int maxMatchedDoc + ) { + this.matchedDocIds = matchedDocIds; + this.remainingPredicateColumns = remainingPredicateColumns; + this.numOfMatchedDocs = numOfMatchedDocs; + this.maxMatchedDoc = maxMatchedDoc; + } + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java new file mode 100644 index 0000000000000..cda3a25b30e53 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; + +import java.util.Map; + +/** + * Query class for querying star tree data structure. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeQueryContext { + + /** + * Star tree field info + * This is used to get the star tree data structure + */ + private final CompositeIndexFieldInfo starTree; + + /** + * Map of field name to a value to be queried for that field + * This is used to filter the data based on the query + */ + private final Map queryMap; + + /** + * Cache for leaf results + * This is used to cache the results for each leaf reader context + * to avoid reading the filtered values from the leaf reader context multiple times + */ + private final FixedBitSet[] starTreeValues; + + public StarTreeQueryContext(CompositeIndexFieldInfo starTree, Map queryMap, int numSegmentsCache) { + this.starTree = starTree; + this.queryMap = queryMap; + if (numSegmentsCache > -1) { + starTreeValues = new FixedBitSet[numSegmentsCache]; + } else { + starTreeValues = null; + } + } + + public CompositeIndexFieldInfo getStarTree() { + return starTree; + } + + public Map getQueryMap() { + return queryMap; + } + + public FixedBitSet[] getStarTreeValues() { + return starTreeValues; + } + + public FixedBitSet getStarTreeValues(LeafReaderContext ctx) { + if (starTreeValues != null) { + return starTreeValues[ctx.ord]; + } + return null; + } + + public void setStarTreeValues(LeafReaderContext ctx, FixedBitSet values) { + if (starTreeValues != null) { + starTreeValues[ctx.ord] = values; + } + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/package-info.java b/server/src/main/java/org/opensearch/search/startree/package-info.java new file mode 100644 index 0000000000000..601a588e54e69 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Star Tree query classes */ +package org.opensearch.search.startree; diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index d35fc6b111c9f..f081cadc1362c 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -109,7 +109,7 @@ protected Codec getCodec() { final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); try { - createMapperService(getExpandedMapping()); + mapperService = createMapperService(getExpandedMapping()); } catch (IOException e) { throw new RuntimeException(e); } @@ -307,7 +307,7 @@ public void testStarTreeDocValuesWithDeletions() throws IOException { directory.close(); } - private XContentBuilder getExpandedMapping() throws IOException { + public static XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); @@ -361,13 +361,13 @@ private XContentBuilder getExpandedMapping() throws IOException { }); } - private XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { + public static XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); buildFields.accept(builder); return builder.endObject().endObject(); } - private void createMapperService(XContentBuilder builder) throws IOException { + public static MapperService createMapperService(XContentBuilder builder) throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) @@ -377,7 +377,7 @@ private void createMapperService(XContentBuilder builder) throws IOException { .build(); IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build(); IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); - mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( + MapperService mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), createTempDir(), settings, @@ -385,5 +385,6 @@ private void createMapperService(XContentBuilder builder) throws IOException { "test" ); mapperService.merge(indexMetadata, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService; } } diff --git a/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java new file mode 100644 index 0000000000000..0c88154ca2b38 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search; + +import org.opensearch.action.OriginalIndices; +import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.Strings; +import org.opensearch.index.IndexService; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests; +import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.internal.AliasFilter; +import org.opensearch.search.internal.ReaderContext; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.internal.ShardSearchRequest; +import org.opensearch.search.startree.StarTreeQueryContext; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; + +public class SearchServiceStarTreeTests extends OpenSearchSingleNodeTestCase { + + public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build()); + setStarTreeIndexSetting("true"); + + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .build(); + CreateIndexRequestBuilder builder = client().admin() + .indices() + .prepareCreate("test") + .setSettings(settings) + .setMapping(StarTreeDocValuesFormatTests.getExpandedMapping()); + createIndex("test", builder); + + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex("test")); + IndexShard indexShard = indexService.getShard(0); + ShardSearchRequest request = new ShardSearchRequest( + OriginalIndices.NONE, + new SearchRequest().allowPartialSearchResults(true), + indexShard.shardId(), + 1, + new AliasFilter(null, Strings.EMPTY_ARRAY), + 1.0f, + -1, + null, + null + ); + + // Case 1: No query or aggregations, should not use star tree + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + assertStarTreeContext(request, sourceBuilder, null, -1); + + // Case 2: MatchAllQuery present but no aggregations, should not use star tree + sourceBuilder = new SearchSourceBuilder().query(new MatchAllQueryBuilder()); + assertStarTreeContext(request, sourceBuilder, null, -1); + + // Case 3: MatchAllQuery and aggregations present, should use star tree + sourceBuilder = new SearchSourceBuilder().size(0) + .query(new MatchAllQueryBuilder()) + .aggregation(AggregationBuilders.max("test").field("field")); + CompositeIndexFieldInfo expectedStarTree = new CompositeIndexFieldInfo( + "startree", + CompositeMappedFieldType.CompositeFieldType.STAR_TREE + ); + Map expectedQueryMap = null; + assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, -1), -1); + + // Case 4: MatchAllQuery and aggregations present, but postFilter specified, should not use star tree + sourceBuilder = new SearchSourceBuilder().size(0) + .query(new MatchAllQueryBuilder()) + .aggregation(AggregationBuilders.max("test").field("field")) + .postFilter(new MatchAllQueryBuilder()); + assertStarTreeContext(request, sourceBuilder, null, -1); + + // Case 5: TermQuery and single aggregation, should use star tree, but not initialize query cache + sourceBuilder = new SearchSourceBuilder().size(0) + .query(new TermQueryBuilder("sndv", 1)) + .aggregation(AggregationBuilders.max("test").field("field")); + expectedQueryMap = Map.of("sndv", 1L); + assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, -1), -1); + + // Case 6: TermQuery and multiple aggregations present, should use star tree & initialize cache + sourceBuilder = new SearchSourceBuilder().size(0) + .query(new TermQueryBuilder("sndv", 1)) + .aggregation(AggregationBuilders.max("test").field("field")) + .aggregation(AggregationBuilders.sum("test2").field("field")); + expectedQueryMap = Map.of("sndv", 1L); + assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, 0), 0); + + // Case 7: No query, metric aggregations present, should use star tree + sourceBuilder = new SearchSourceBuilder().size(0).aggregation(AggregationBuilders.max("test").field("field")); + assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, null, -1), -1); + + setStarTreeIndexSetting(null); + } + + private void setStarTreeIndexSetting(String value) throws IOException { + client().admin() + .cluster() + .prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), value).build()) + .execute(); + } + + private void assertStarTreeContext( + ShardSearchRequest request, + SearchSourceBuilder sourceBuilder, + StarTreeQueryContext expectedContext, + int expectedCacheUsage + ) throws IOException { + request.source(sourceBuilder); + SearchService searchService = getInstanceFromNode(SearchService.class); + try (ReaderContext reader = searchService.createOrGetReaderContext(request, false)) { + SearchContext context = searchService.createContext(reader, request, null, true); + StarTreeQueryContext actualContext = context.getStarTreeQueryContext(); + + if (expectedContext == null) { + assertThat(context.getStarTreeQueryContext(), nullValue()); + } else { + assertThat(actualContext, notNullValue()); + assertEquals(expectedContext.getStarTree().getType(), actualContext.getStarTree().getType()); + assertEquals(expectedContext.getStarTree().getField(), actualContext.getStarTree().getField()); + assertEquals(expectedContext.getQueryMap(), actualContext.getQueryMap()); + if (expectedCacheUsage > -1) { + assertEquals(expectedCacheUsage, actualContext.getStarTreeValues().length); + } else { + assertNull(actualContext.getStarTreeValues()); + } + } + searchService.doStop(); + } + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java new file mode 100644 index 0000000000000..0327bd9990784 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java @@ -0,0 +1,317 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.startree; + +import com.carrotsearch.randomizedtesting.RandomizedTest; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; +import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.opensearch.search.aggregations.metrics.InternalAvg; +import org.opensearch.search.aggregations.metrics.InternalMax; +import org.opensearch.search.aggregations.metrics.InternalMin; +import org.opensearch.search.aggregations.metrics.InternalSum; +import org.opensearch.search.aggregations.metrics.InternalValueCount; +import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.opensearch.search.aggregations.metrics.MinAggregationBuilder; +import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import static org.opensearch.search.aggregations.AggregationBuilders.avg; +import static org.opensearch.search.aggregations.AggregationBuilders.count; +import static org.opensearch.search.aggregations.AggregationBuilders.max; +import static org.opensearch.search.aggregations.AggregationBuilders.min; +import static org.opensearch.search.aggregations.AggregationBuilders.sum; +import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS; + +public class MetricAggregatorTests extends AggregatorTestCase { + + private static final String FIELD_NAME = "field"; + private static final NumberFieldMapper.NumberType DEFAULT_FIELD_TYPE = NumberFieldMapper.NumberType.LONG; + private static final MappedFieldType DEFAULT_MAPPED_FIELD = new NumberFieldMapper.NumberFieldType(FIELD_NAME, DEFAULT_FIELD_TYPE); + + @Before + public void setup() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build()); + } + + @After + public void teardown() throws IOException { + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + protected Codec getCodec() { + final Logger testLogger = LogManager.getLogger(MetricAggregatorTests.class); + MapperService mapperService; + try { + mapperService = StarTreeDocValuesFormatTests.createMapperService(StarTreeDocValuesFormatTests.getExpandedMapping()); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, testLogger); + } + + public void testStarTreeDocValues() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setCodec(getCodec()); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + + Random random = RandomizedTest.getRandom(); + int totalDocs = 100; + final String SNDV = "sndv"; + final String DV = "dv"; + int val; + + List docs = new ArrayList<>(); + // Index 100 random documents + for (int i = 0; i < totalDocs; i++) { + Document doc = new Document(); + if (random.nextBoolean()) { + val = random.nextInt(10) - 5; // Random long between -5 and 4 + doc.add(new SortedNumericDocValuesField(SNDV, val)); + } + if (random.nextBoolean()) { + val = random.nextInt(20) - 10; // Random long between -10 and 9 + doc.add(new SortedNumericDocValuesField(DV, val)); + } + if (random.nextBoolean()) { + val = random.nextInt(50); // Random long between 0 and 49 + doc.add(new SortedNumericDocValuesField(FIELD_NAME, val)); + } + iw.addDocument(doc); + docs.add(doc); + } + + if (randomBoolean()) { + iw.forceMerge(1); + } + iw.close(); + + DirectoryReader ir = DirectoryReader.open(directory); + initValuesSourceRegistry(); + LeafReaderContext context = ir.leaves().get(0); + + SegmentReader reader = Lucene.segmentReader(context.reader()); + IndexSearcher indexSearcher = newSearcher(reader, false, false); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + CompositeIndexFieldInfo starTree = compositeIndexFields.get(0); + + SumAggregationBuilder sumAggregationBuilder = sum("_name").field(FIELD_NAME); + MaxAggregationBuilder maxAggregationBuilder = max("_name").field(FIELD_NAME); + MinAggregationBuilder minAggregationBuilder = min("_name").field(FIELD_NAME); + ValueCountAggregationBuilder valueCountAggregationBuilder = count("_name").field(FIELD_NAME); + AvgAggregationBuilder avgAggregationBuilder = avg("_name").field(FIELD_NAME); + + List supportedDimensions = new LinkedList<>(); + supportedDimensions.add(new NumericDimension(SNDV)); + supportedDimensions.add(new NumericDimension(DV)); + + Query query = new MatchAllDocsQuery(); + // match-all query + QueryBuilder queryBuilder = null; // no predicates + testCase( + indexSearcher, + query, + queryBuilder, + sumAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalSum::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + maxAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMax::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + minAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMin::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + valueCountAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalValueCount::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + avgAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalAvg::getValue) + ); + + // Numeric-terms query + for (int cases = 0; cases < 100; cases++) { + String queryField; + long queryValue; + if (randomBoolean()) { + queryField = SNDV; + queryValue = random.nextInt(10); + } else { + queryField = DV; + queryValue = random.nextInt(20) - 15; + } + + query = SortedNumericDocValuesField.newSlowExactQuery(queryField, queryValue); + queryBuilder = new TermQueryBuilder(queryField, queryValue); + + testCase( + indexSearcher, + query, + queryBuilder, + sumAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalSum::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + maxAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMax::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + minAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMin::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + valueCountAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalValueCount::getValue) + ); + testCase( + indexSearcher, + query, + queryBuilder, + avgAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalAvg::getValue) + ); + } + + ir.close(); + directory.close(); + } + + BiConsumer verifyAggregation(Function valueExtractor) { + return (expectedAggregation, actualAggregation) -> assertEquals( + valueExtractor.apply(expectedAggregation).doubleValue(), + valueExtractor.apply(actualAggregation).doubleValue(), + 0.0f + ); + } + + private void testCase( + IndexSearcher searcher, + Query query, + QueryBuilder queryBuilder, + T aggBuilder, + CompositeIndexFieldInfo starTree, + List supportedDimensions, + BiConsumer verify + ) throws IOException { + V starTreeAggregation = searchAndReduceStarTree( + createIndexSettings(), + searcher, + query, + queryBuilder, + aggBuilder, + starTree, + supportedDimensions, + DEFAULT_MAX_BUCKETS, + false, + DEFAULT_MAPPED_FIELD + ); + V expectedAggregation = searchAndReduceStarTree( + createIndexSettings(), + searcher, + query, + queryBuilder, + aggBuilder, + null, + null, + DEFAULT_MAX_BUCKETS, + false, + DEFAULT_MAPPED_FIELD + ); + verify.accept(expectedAggregation, starTreeAggregation); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java new file mode 100644 index 0000000000000..f8eb71a40319a --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java @@ -0,0 +1,319 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.startree; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; +import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.startree.StarTreeFilter; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests.topMapping; + +public class StarTreeFilterTests extends AggregatorTestCase { + + private static final String FIELD_NAME = "field"; + private static final String SNDV = "sndv"; + private static final String SDV = "sdv"; + private static final String DV = "dv"; + + @Before + public void setup() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build()); + } + + @After + public void teardown() throws IOException { + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + protected Codec getCodec(int maxLeafDoc, boolean skipStarNodeCreationForSDVDimension) { + final Logger testLogger = LogManager.getLogger(StarTreeFilterTests.class); + MapperService mapperService; + try { + mapperService = StarTreeDocValuesFormatTests.createMapperService( + getExpandedMapping(maxLeafDoc, skipStarNodeCreationForSDVDimension) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, testLogger); + } + + public void testStarTreeFilterWithNoDocsInSVDField() throws IOException { + testStarTreeFilter(5, true); + } + + public void testStarTreeFilterWithDocsInSVDFieldButNoStarNode() throws IOException { + testStarTreeFilter(10, false); + } + + private void testStarTreeFilter(int maxLeafDoc, boolean skipStarNodeCreationForSDVDimension) throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setCodec(getCodec(maxLeafDoc, skipStarNodeCreationForSDVDimension)); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + int totalDocs = 100; + + List docs = new ArrayList<>(); + for (int i = 0; i < totalDocs; i++) { + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField(SNDV, i)); + doc.add(new SortedNumericDocValuesField(DV, 2 * i)); + doc.add(new SortedNumericDocValuesField(FIELD_NAME, 3 * i)); + if (skipStarNodeCreationForSDVDimension) { + // adding SDV field only star node creation is skipped for SDV dimension + doc.add(new SortedNumericDocValuesField(SDV, 4 * i)); + } + iw.addDocument(doc); + docs.add(doc); + } + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = DirectoryReader.open(directory); + initValuesSourceRegistry(); + LeafReaderContext context = ir.leaves().get(0); + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + + long starTreeDocCount, docCount; + + // assert that all documents are included if no filters are given + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(), context); + docCount = getDocCount(docs, Map.of()); + assertEquals(totalDocs, starTreeDocCount); + assertEquals(docCount, starTreeDocCount); + + // single filter - matches docs + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L), context); + docCount = getDocCount(docs, Map.of(SNDV, 0L)); + assertEquals(1, docCount); + assertEquals(docCount, starTreeDocCount); + + // single filter on 3rd field in ordered dimension - matches docs + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(DV, 0L), context); + docCount = getDocCount(docs, Map.of(DV, 0L)); + assertEquals(1, docCount); + assertEquals(docCount, starTreeDocCount); + + // single filter - does not match docs + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 101L), context); + docCount = getDocCount(docs, Map.of(SNDV, 101L)); + assertEquals(0, docCount); + assertEquals(docCount, starTreeDocCount); + + // single filter on 3rd field in ordered dimension - does not match docs + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(DV, -101L), context); + docCount = getDocCount(docs, Map.of(SNDV, -101L)); + assertEquals(0, docCount); + assertEquals(docCount, starTreeDocCount); + + // multiple filters - matches docs + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, 0L), context); + docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, 0L)); + assertEquals(1, docCount); + assertEquals(docCount, starTreeDocCount); + + // no document should match the filter + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, -11L), context); + docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, -11L)); + assertEquals(0, docCount); + assertEquals(docCount, starTreeDocCount); + + // Only the first filter should match some documents, second filter matches none + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, -100L), context); + docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, -100L)); + assertEquals(0, docCount); + assertEquals(docCount, starTreeDocCount); + + // non-dimension fields in filter - should throw IllegalArgumentException + expectThrows( + IllegalArgumentException.class, + () -> getDocCountFromStarTree(starTreeDocValuesReader, Map.of(FIELD_NAME, 0L), context) + ); + + if (skipStarNodeCreationForSDVDimension == true) { + // Documents are not indexed + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SDV, 4L), context); + docCount = getDocCount(docs, Map.of(SDV, 4L)); + assertEquals(1, docCount); + assertEquals(docCount, starTreeDocCount); + } else { + // Documents are indexed + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SDV, 4L), context); + docCount = getDocCount(docs, Map.of(SDV, 4L)); + assertEquals(0, docCount); + assertEquals(docCount, starTreeDocCount); + } + + ir.close(); + directory.close(); + } + + // Counts the documents having field SNDV & applied filters + private long getDocCount(List documents, Map filters) { + long count = 0; + for (Document doc : documents) { + // Check if SNDV field is present + IndexableField sndvField = doc.getField(SNDV); + if (sndvField == null) continue; // Skip if SNDV is not present + + // Apply filters if provided + if (!filters.isEmpty()) { + boolean matches = filters.entrySet().stream().allMatch(entry -> { + IndexableField field = doc.getField(entry.getKey()); + return field != null && field.numericValue().longValue() == entry.getValue(); + }); + if (!matches) continue; + } + + // Increment count if the document passes all conditions + count++; + } + return count; + } + + // Returns count of documents in the star tree having field SNDV & applied filters + private long getDocCountFromStarTree(CompositeIndexReader starTreeDocValuesReader, Map filters, LeafReaderContext context) + throws IOException { + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + CompositeIndexFieldInfo starTree = compositeIndexFields.get(0); + StarTreeValues starTreeValues = StarTreeQueryHelper.getStarTreeValues(context, starTree); + FixedBitSet filteredValues = StarTreeFilter.getStarTreeResult(starTreeValues, filters); + + SortedNumericStarTreeValuesIterator valuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues.getMetricValuesIterator( + StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTree.getField(), + SNDV, + MetricStat.VALUE_COUNT.getTypeName() + ) + ); + + long docCount = 0; + int numBits = filteredValues.length(); + if (numBits > 0) { + for (int bit = filteredValues.nextSetBit(0); bit != DocIdSetIterator.NO_MORE_DOCS; bit = (bit + 1 < numBits) + ? filteredValues.nextSetBit(bit + 1) + : DocIdSetIterator.NO_MORE_DOCS) { + + // Assert that we can advance to the document ID in the values iterator + boolean canAdvance = valuesIterator.advanceExact(bit); + assert canAdvance : "Cannot advance to document ID " + bit + " in values iterator."; + + // Iterate over values for the current document ID + for (int i = 0, count = valuesIterator.entryValueCount(); i < count; i++) { + long value = valuesIterator.nextValue(); + // Assert that the value is as expected using the provided consumer + docCount += value; + } + } + } + return docCount; + } + + public static XContentBuilder getExpandedMapping(int maxLeafDocs, boolean skipStarNodeCreationForSDVDimension) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("max_leaf_docs", maxLeafDocs); + if (skipStarNodeCreationForSDVDimension) { + b.startArray("skip_star_node_creation_for_dimensions"); + b.value("sdv"); + b.endArray(); + } + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "sndv"); + b.endObject(); + b.startObject(); + b.field("name", "sdv"); + b.endObject(); + b.startObject(); + b.field("name", "dv"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "field"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.startObject(); + b.field("name", "sndv"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("sndv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("sdv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("dv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("field"); + b.field("type", "integer"); + b.endObject(); + b.endObject(); + }); + } +} diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index 4abd7fbea9cff..e1728c4476699 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -91,11 +91,16 @@ import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.bitset.BitsetFilterCache.Listener; import org.opensearch.index.cache.query.DisabledQueryCache; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.CompletionFieldMapper; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.ConstantKeywordFieldMapper; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DateFieldMapper; @@ -117,6 +122,7 @@ import org.opensearch.index.mapper.RangeType; import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.mapper.TextFieldMapper; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; @@ -135,12 +141,14 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.aggregations.support.ValuesSourceType; +import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.fetch.subphase.FetchDocValuesPhase; import org.opensearch.search.fetch.subphase.FetchSourcePhase; import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.search.internal.SearchContext; import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.search.startree.StarTreeQueryContext; import org.opensearch.test.InternalAggregationTestCase; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; @@ -155,6 +163,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; @@ -331,6 +340,35 @@ protected A createAggregator(AggregationBuilder aggregati return aggregator; } + protected CountingAggregator createCountingAggregator( + Query query, + QueryBuilder queryBuilder, + AggregationBuilder aggregationBuilder, + IndexSearcher indexSearcher, + IndexSettings indexSettings, + CompositeIndexFieldInfo starTree, + List supportedDimensions, + MultiBucketConsumer bucketConsumer, + MappedFieldType... fieldTypes + ) throws IOException { + SearchContext searchContext; + if (starTree != null) { + searchContext = createSearchContextWithStarTreeContext( + indexSearcher, + indexSettings, + query, + queryBuilder, + starTree, + supportedDimensions, + bucketConsumer, + fieldTypes + ); + } else { + searchContext = createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, fieldTypes); + } + return new CountingAggregator(new AtomicInteger(), createAggregator(aggregationBuilder, searchContext)); + } + /** * Create a {@linkplain SearchContext} for testing an {@link Aggregator}. */ @@ -344,6 +382,49 @@ protected SearchContext createSearchContext( return createSearchContext(indexSearcher, indexSettings, query, bucketConsumer, new NoneCircuitBreakerService(), fieldTypes); } + protected SearchContext createSearchContextWithStarTreeContext( + IndexSearcher indexSearcher, + IndexSettings indexSettings, + Query query, + QueryBuilder queryBuilder, + CompositeIndexFieldInfo starTree, + List supportedDimensions, + MultiBucketConsumer bucketConsumer, + MappedFieldType... fieldTypes + ) throws IOException { + SearchContext searchContext = createSearchContext( + indexSearcher, + indexSettings, + query, + bucketConsumer, + new NoneCircuitBreakerService(), + fieldTypes + ); + + // Mock SearchContextAggregations + SearchContextAggregations searchContextAggregations = mock(SearchContextAggregations.class); + AggregatorFactories aggregatorFactories = mock(AggregatorFactories.class); + when(searchContext.aggregations()).thenReturn(searchContextAggregations); + when(searchContextAggregations.factories()).thenReturn(aggregatorFactories); + when(aggregatorFactories.getFactories()).thenReturn(new AggregatorFactory[] {}); + + CompositeDataCubeFieldType compositeMappedFieldType = mock(CompositeDataCubeFieldType.class); + when(compositeMappedFieldType.name()).thenReturn(starTree.getField()); + when(compositeMappedFieldType.getCompositeIndexType()).thenReturn(starTree.getType()); + Set compositeFieldTypes = Set.of(compositeMappedFieldType); + + when((compositeMappedFieldType).getDimensions()).thenReturn(supportedDimensions); + MapperService mapperService = mock(MapperService.class); + when(mapperService.getCompositeFieldTypes()).thenReturn(compositeFieldTypes); + when(searchContext.mapperService()).thenReturn(mapperService); + + SearchSourceBuilder sb = new SearchSourceBuilder().query(queryBuilder); + StarTreeQueryContext starTreeQueryContext = StarTreeQueryHelper.getStarTreeQueryContext(searchContext, sb); + + when(searchContext.getStarTreeQueryContext()).thenReturn(starTreeQueryContext); + return searchContext; + } + protected SearchContext createSearchContext( IndexSearcher indexSearcher, IndexSettings indexSettings, @@ -651,6 +732,67 @@ protected A searchAndReduc return internalAgg; } + protected A searchAndReduceStarTree( + IndexSettings indexSettings, + IndexSearcher searcher, + Query query, + QueryBuilder queryBuilder, + AggregationBuilder builder, + CompositeIndexFieldInfo compositeIndexFieldInfo, + List supportedDimensions, + int maxBucket, + boolean hasNested, + MappedFieldType... fieldTypes + ) throws IOException { + query = query.rewrite(searcher); + final IndexReaderContext ctx = searcher.getTopReaderContext(); + final PipelineTree pipelines = builder.buildPipelineTree(); + List aggs = new ArrayList<>(); + if (hasNested) { + query = Queries.filtered(query, Queries.newNonNestedFilter()); + } + + MultiBucketConsumer bucketConsumer = new MultiBucketConsumer( + maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) + ); + CountingAggregator countingAggregator = createCountingAggregator( + query, + queryBuilder, + builder, + searcher, + indexSettings, + compositeIndexFieldInfo, + supportedDimensions, + bucketConsumer, + fieldTypes + ); + + countingAggregator.preCollection(); + searcher.search(query, countingAggregator); + countingAggregator.postCollection(); + aggs.add(countingAggregator.buildTopLevel()); + if (compositeIndexFieldInfo != null) { + assertEquals(0, countingAggregator.collectCounter.get()); + } + + MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer( + maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) + ); + InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( + countingAggregator.context().bigArrays(), + getMockScriptService(), + reduceBucketConsumer, + pipelines + ); + + @SuppressWarnings("unchecked") + A internalAgg = (A) aggs.get(0).reduce(aggs, context); + doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + return internalAgg; + } + protected void doAssertReducedMultiBucketConsumer(Aggregation agg, MultiBucketConsumerService.MultiBucketConsumer bucketConsumer) { InternalAggregationTestCase.assertMultiBucketConsumer(agg, bucketConsumer); } From 19824275fe78014249143724c5b84a6211e6e642 Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:34:14 +0530 Subject: [PATCH 269/357] URI path filtering support in cluster stats API (#15938) * URI path filtering support in cluster stats API Signed-off-by: Swetha Guptha --- CHANGELOG.md | 3 +- .../opensearch/upgrades/ClusterStatsIT.java | 67 ++++ .../admin/cluster/stats/ClusterStatsIT.java | 376 ++++++++++++++++++ .../cluster/stats/ClusterStatsIndices.java | 109 +++-- .../cluster/stats/ClusterStatsNodes.java | 105 +++-- .../cluster/stats/ClusterStatsRequest.java | 142 +++++++ .../stats/ClusterStatsRequestBuilder.java | 20 + .../cluster/stats/ClusterStatsResponse.java | 51 ++- .../stats/TransportClusterStatsAction.java | 133 +++++-- .../index/cache/query/QueryCacheStats.java | 3 + .../index/fielddata/FieldDataStats.java | 3 + .../admin/cluster/RestClusterStatsAction.java | 135 ++++++- .../stats/ClusterStatsResponseTests.java | 281 +++++++++++++ .../cluster/RestClusterStatsActionTests.java | 171 ++++++++ 14 files changed, 1483 insertions(+), 116 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java create mode 100644 server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java create mode 100644 server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b65a6cd70dc..832871453028b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) -- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387) +- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)) +- URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) - [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) ### Dependencies diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java new file mode 100644 index 0000000000000..1c5f35db8ec46 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.upgrades; + +import org.opensearch.Version; +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ClusterStatsIT extends AbstractRollingTestCase { + + private final List nodeStatsMetrics = List.of("os", "process", "jvm", "fs", "plugins", "ingest", "network_types", "discovery_types", "packaging_types"); + + private final List indicesStatsMetrics = List.of("shards", "docs", "store", "fielddata", "query_cache", "completion", "segments", "analysis", "mappings"); + + public void testClusterStats() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cluster/stats")); + validateClusterStatsWithFilterResponse(response, nodeStatsMetrics, indicesStatsMetrics); + if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_3_0_0) || ( + CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_3_0_0))) { + response = client().performRequest(new Request("GET", "/_cluster/stats/os/nodes/_all")); + validateClusterStatsWithFilterResponse(response, List.of("os"), Collections.emptyList()); + response = client().performRequest(new Request("GET", "/_cluster/stats/indices/mappings/nodes/_all")); + validateClusterStatsWithFilterResponse(response, Collections.emptyList(), List.of("mappings")); + response = client().performRequest(new Request("GET", "/_cluster/stats/os,indices/mappings/nodes/_all")); + validateClusterStatsWithFilterResponse(response, List.of("os"), List.of("mappings")); + } + } + + private void validateClusterStatsWithFilterResponse(Response response, List requestedNodesStatsMetrics, List requestedIndicesStatsMetrics) throws IOException { + assertEquals(200, response.getStatusLine().getStatusCode()); + Map entity = entityAsMap(response); + if (requestedNodesStatsMetrics != null && !requestedNodesStatsMetrics.isEmpty()) { + assertTrue(entity.containsKey("nodes")); + Map nodesStats = (Map) entity.get("nodes"); + for (String metric : nodeStatsMetrics) { + if (requestedNodesStatsMetrics.contains(metric)) { + assertTrue(nodesStats.containsKey(metric)); + } else { + assertFalse(nodesStats.containsKey(metric)); + } + } + } + + if (requestedIndicesStatsMetrics != null && !requestedIndicesStatsMetrics.isEmpty()) { + assertTrue(entity.containsKey("indices")); + Map indicesStats = (Map) entity.get("indices"); + for (String metric : indicesStatsMetrics) { + if (requestedIndicesStatsMetrics.contains(metric)) { + assertTrue(indicesStats.containsKey(metric)); + } else { + assertFalse(indicesStats.containsKey(metric)); + } + } + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java index f23cdbb50b37a..5f00ba35c7b69 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -37,6 +37,9 @@ import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; +import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; import org.opensearch.client.Requests; import org.opensearch.cluster.health.ClusterHealthStatus; @@ -44,6 +47,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.monitor.os.OsStats; import org.opensearch.node.NodeRoleSettings; import org.opensearch.test.OpenSearchIntegTestCase; @@ -230,6 +234,7 @@ public void testIndicesShardStatsWithoutNodeLevelAggregations() { } public void testIndicesShardStatsWithNodeLevelAggregations() { + internalCluster().startNode(); ensureGreen(); ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(true).get(); @@ -317,6 +322,53 @@ public void testValuesSmokeScreen() throws IOException, ExecutionException, Inte assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); } + public void testValuesSmokeScreenWithNodeStatsAndIndicesStatsMetricsFilter() throws IOException, ExecutionException, + InterruptedException { + internalCluster().startNodes(randomIntBetween(1, 3)); + index("test1", "type", "1", "f", "f"); + + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .computeAllMetrics(false) + .requestMetrics(Set.of(Metric.values())) + .indexMetrics(Set.of(IndexMetric.values())) + .get(); + String msg = response.toString(); + assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L)); // 1 Jan 2000 + assertThat(msg, response.indicesStats.getStore().getSizeInBytes(), Matchers.greaterThan(0L)); + + assertThat(msg, response.nodesStats.getFs().getTotal().getBytes(), Matchers.greaterThan(0L)); + assertThat(msg, response.nodesStats.getJvm().getVersions().size(), Matchers.greaterThan(0)); + + assertThat(msg, response.nodesStats.getVersions().size(), Matchers.greaterThan(0)); + assertThat(msg, response.nodesStats.getVersions().contains(Version.CURRENT), Matchers.equalTo(true)); + assertThat(msg, response.nodesStats.getPlugins().size(), Matchers.greaterThanOrEqualTo(0)); + + assertThat(msg, response.nodesStats.getProcess().count, Matchers.greaterThan(0)); + // 0 happens when not supported on platform + assertThat(msg, response.nodesStats.getProcess().getAvgOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(0L)); + // these can be -1 if not supported on platform + assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L)); + assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L)); + + NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().addMetric(OS.metricName()).get(); + long total = 0; + long free = 0; + long used = 0; + for (NodeStats nodeStats : nodesStatsResponse.getNodes()) { + total += nodeStats.getOs().getMem().getTotal().getBytes(); + free += nodeStats.getOs().getMem().getFree().getBytes(); + used += nodeStats.getOs().getMem().getUsed().getBytes(); + } + assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().getBytes()); + assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().getBytes()); + assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().getBytes()); + assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent()); + assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent()); + } + public void testAllocatedProcessors() throws Exception { // start one node with 7 processors. internalCluster().startNode(Settings.builder().put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 7).build()); @@ -384,6 +436,43 @@ public void testFieldTypes() { } } + public void testFieldTypesWithMappingsFilter() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse response = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .computeAllMetrics(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS)) + .get(); + assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN)); + assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty()); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + client().admin() + .indices() + .prepareCreate("test2") + .setMapping( + "{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," + + "\"eggplant\":{\"type\":\"integer\"}}}}}" + ) + .get(); + response = client().admin().cluster().prepareClusterStats().useAggregatedNodeLevelResponses(randomBoolean()).get(); + assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3)); + Set stats = response.getIndicesStats().getMappings().getFieldTypeStats(); + for (IndexFeatureStats stat : stats) { + if (stat.getName().equals("integer")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } else if (stat.getName().equals("keyword")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(3)); + } else if (stat.getName().equals("object")) { + assertThat(stat.getCount(), greaterThanOrEqualTo(1)); + } + } + } + public void testNodeRolesWithMasterLegacySettings() throws ExecutionException, InterruptedException { int total = 1; Settings legacyMasterSettings = Settings.builder() @@ -505,6 +594,293 @@ public void testNodeRolesWithDataNodeLegacySettings() throws ExecutionException, assertEquals(expectedNodesRoles, Set.of(getNodeRoles(client, 0), getNodeRoles(client, 1))); } + public void testClusterStatsWithNodeMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithAllNodeStatsMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(ClusterStatsNodes.NODE_STATS_METRICS) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithAllNodeStatsMetrics); + assertNotNull(statsResponseWithAllNodeStatsMetrics.getNodesStats()); + assertNull(statsResponseWithAllNodeStatsMetrics.getIndicesStats()); + validateNodeStatsOutput(ClusterStatsNodes.NODE_STATS_METRICS, statsResponseWithAllNodeStatsMetrics); + assertEquals( + response.getNodesStats().getCounts().getTotal(), + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getTotal() + ); + assertEquals( + response.getNodesStats().getCounts().getRoles(), + statsResponseWithAllNodeStatsMetrics.getNodesStats().getCounts().getRoles() + ); + assertEquals(response.getNodesStats().getVersions(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getVersions()); + assertEquals(response.getNodesStats().getPlugins(), statsResponseWithAllNodeStatsMetrics.getNodesStats().getPlugins()); + } + + public void testClusterStatsWithIndicesOnlyMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithIndicesRequestMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.values())) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithIndicesRequestMetrics); + assertNull(statsResponseWithIndicesRequestMetrics.getNodesStats()); + assertNotNull(statsResponseWithIndicesRequestMetrics.getIndicesStats()); + validateIndicesStatsOutput(Set.of(IndexMetric.values()), statsResponseWithIndicesRequestMetrics); + } + + public void testClusterStatsWithSelectiveNodeMetricAndIndexMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); + client().index(indexRequest); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.OS, Metric.FS, Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNotNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateNodeStatsOutput(Set.of(Metric.FS, Metric.OS), statsResponseWithAllIndicesMetrics); + validateIndicesStatsOutput( + Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE), + statsResponseWithAllIndicesMetrics + ); + assertEquals(response.getIndicesStats().getFieldData(), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData()); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithAllIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals( + response.getIndicesStats().getShards().getTotal(), + statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getTotal() + ); + assertEquals( + response.getIndicesStats().getShards().getPrimaries(), + statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries() + ); + } + + public void testClusterStatsWithMappingsAndAnalysisStatsIndexMetricsFilter() { + internalCluster().startNode(); + ensureGreen(); + + client().admin().indices().prepareCreate("test1").setMapping("{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}").get(); + IndexRequest indexRequest = new IndexRequest("test1").id("doc_id").source(Map.of("test_type", "metrics_filter")); + client().index(indexRequest); + + ClusterStatsRequestBuilder clusterStatsRequestBuilder = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()); + assertTrue(clusterStatsRequestBuilder.request().computeAllMetrics()); + + ClusterStatsResponse response = clusterStatsRequestBuilder.get(); + assertNotNull(response); + assertNotNull(response.getNodesStats()); + assertNotNull(response.getIndicesStats()); + + ClusterStatsResponse statsResponseWithSpecificIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithSpecificIndicesMetrics); + assertNull(statsResponseWithSpecificIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithSpecificIndicesMetrics.getIndicesStats()); + validateIndicesStatsOutput(Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS), statsResponseWithSpecificIndicesMetrics); + assertEquals(response.getIndicesStats().getIndexCount(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getIndexCount()); + assertEquals(response.getIndicesStats().getMappings(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getMappings()); + assertEquals(response.getIndicesStats().getAnalysis(), statsResponseWithSpecificIndicesMetrics.getIndicesStats().getAnalysis()); + } + + public void testClusterStatsWithIndexMetricWithDocsFilter() throws IOException { + internalCluster().startNode(); + createIndex("test1"); + + client().prepareIndex("test1").setId(Integer.toString(1)).setSource("field1", "value1").execute().actionGet(); + client().prepareIndex("test1").setId(Integer.toString(2)).setSource("field2", "value2").execute().actionGet(); + refreshAndWaitForReplication(); + + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.DOCS)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateIndicesStatsOutput(Set.of(IndexMetric.DOCS), statsResponseWithAllIndicesMetrics); + assertEquals(2, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); + assertTrue(statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getAverageSizeInBytes() > 0); + } + + public void testClusterStatsWithSelectiveMetricsFilterAndNoIndex() { + internalCluster().startNode(); + ensureGreen(); + ClusterStatsResponse statsResponseWithAllIndicesMetrics = client().admin() + .cluster() + .prepareClusterStats() + .useAggregatedNodeLevelResponses(randomBoolean()) + .requestMetrics(Set.of(Metric.OS, Metric.FS, Metric.INDICES)) + .indexMetrics(Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE)) + .computeAllMetrics(false) + .get(); + assertNotNull(statsResponseWithAllIndicesMetrics); + assertNotNull(statsResponseWithAllIndicesMetrics.getNodesStats()); + assertNotNull(statsResponseWithAllIndicesMetrics.getIndicesStats()); + validateNodeStatsOutput(Set.of(Metric.FS, Metric.OS), statsResponseWithAllIndicesMetrics); + validateIndicesStatsOutput( + Set.of(IndexMetric.FIELDDATA, IndexMetric.SHARDS, IndexMetric.SEGMENTS, IndexMetric.DOCS, IndexMetric.STORE), + statsResponseWithAllIndicesMetrics + ); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getIndices()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getTotal()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getShards().getPrimaries()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getDeleted()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getDocs().getTotalSizeInBytes()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getStore().getSizeInBytes()); + assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getStore().getReservedSize()); + assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getMemorySize()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getEvictions()); + assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getFields()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getCount()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getIndexWriterMemoryInBytes()); + assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getVersionMapMemoryInBytes()); + } + + private void validateNodeStatsOutput(Set expectedMetrics, ClusterStatsResponse clusterStatsResponse) { + // Ingest, network types, discovery types and packaging types stats are not included here as they don't have a get method exposed. + Set NodeMetrics = Set.of(Metric.OS, Metric.JVM, Metric.FS, Metric.PROCESS, Metric.PLUGINS); + for (Metric metric : NodeMetrics) { + Object object = null; + switch (metric) { + case OS: + object = clusterStatsResponse.getNodesStats().getOs(); + break; + case JVM: + object = clusterStatsResponse.getNodesStats().getJvm(); + break; + case FS: + object = clusterStatsResponse.getNodesStats().getFs(); + break; + case PROCESS: + object = clusterStatsResponse.getNodesStats().getProcess(); + break; + case PLUGINS: + object = clusterStatsResponse.getNodesStats().getPlugins(); + break; + } + if (expectedMetrics.contains(metric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + + private void validateIndicesStatsOutput( + Set expectedMetrics, + ClusterStatsResponse clusterStatsResponse + ) { + for (IndexMetric indexMetric : IndexMetric.values()) { + Object object = null; + switch (indexMetric) { + case SHARDS: + object = clusterStatsResponse.getIndicesStats().getShards(); + break; + case DOCS: + object = clusterStatsResponse.getIndicesStats().getDocs(); + break; + case STORE: + object = clusterStatsResponse.getIndicesStats().getStore(); + break; + case FIELDDATA: + object = clusterStatsResponse.getIndicesStats().getFieldData(); + break; + case QUERY_CACHE: + object = clusterStatsResponse.getIndicesStats().getQueryCache(); + break; + case COMPLETION: + object = clusterStatsResponse.getIndicesStats().getCompletion(); + break; + case SEGMENTS: + object = clusterStatsResponse.getIndicesStats().getSegments(); + break; + case ANALYSIS: + object = clusterStatsResponse.getIndicesStats().getAnalysis(); + break; + case MAPPINGS: + object = clusterStatsResponse.getIndicesStats().getMappings(); + break; + } + if (expectedMetrics.contains(indexMetric)) { + assertNotNull(object); + } else { + assertNull(object); + } + } + } + private Map getExpectedCounts( int dataRoleCount, int masterRoleCount, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java index 03a73f45ffe81..9ebe36531c208 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.cluster.stats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.xcontent.ToXContentFragment; @@ -47,6 +48,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; /** * Cluster Stats per index @@ -68,14 +70,23 @@ public class ClusterStatsIndices implements ToXContentFragment { private MappingStats mappings; public ClusterStatsIndices(List nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats) { - Map countsPerIndex = new HashMap<>(); + this(Set.of(IndexMetric.values()), nodeResponses, mappingStats, analysisStats); + + } - this.docs = new DocsStats(); - this.store = new StoreStats(); - this.fieldData = new FieldDataStats(); - this.queryCache = new QueryCacheStats(); - this.completion = new CompletionStats(); - this.segments = new SegmentsStats(); + public ClusterStatsIndices( + Set indicesMetrics, + List nodeResponses, + MappingStats mappingStats, + AnalysisStats analysisStats + ) { + Map countsPerIndex = new HashMap<>(); + this.docs = indicesMetrics.contains(IndexMetric.DOCS) ? new DocsStats() : null; + this.store = indicesMetrics.contains(IndexMetric.STORE) ? new StoreStats() : null; + this.fieldData = indicesMetrics.contains(IndexMetric.FIELDDATA) ? new FieldDataStats() : null; + this.queryCache = indicesMetrics.contains(IndexMetric.QUERY_CACHE) ? new QueryCacheStats() : null; + this.completion = indicesMetrics.contains(IndexMetric.COMPLETION) ? new CompletionStats() : null; + this.segments = indicesMetrics.contains(IndexMetric.SEGMENTS) ? new SegmentsStats() : null; for (ClusterStatsNodeResponse r : nodeResponses) { // Aggregated response from the node @@ -92,12 +103,24 @@ public ClusterStatsIndices(List nodeResponses, Mapping } } - docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); - store.add(r.getAggregatedNodeLevelStats().commonStats.store); - fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); - queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); - completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); - segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); + if (indicesMetrics.contains(IndexMetric.DOCS)) { + docs.add(r.getAggregatedNodeLevelStats().commonStats.docs); + } + if (indicesMetrics.contains(IndexMetric.STORE)) { + store.add(r.getAggregatedNodeLevelStats().commonStats.store); + } + if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { + fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData); + } + if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { + queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache); + } + if (indicesMetrics.contains(IndexMetric.COMPLETION)) { + completion.add(r.getAggregatedNodeLevelStats().commonStats.completion); + } + if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { + segments.add(r.getAggregatedNodeLevelStats().commonStats.segments); + } } else { // Default response from the node for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -113,21 +136,35 @@ public ClusterStatsIndices(List nodeResponses, Mapping if (shardStats.getShardRouting().primary()) { indexShardStats.primaries++; - docs.add(shardCommonStats.docs); + if (indicesMetrics.contains(IndexMetric.DOCS)) { + docs.add(shardCommonStats.docs); + } + } + if (indicesMetrics.contains(IndexMetric.STORE)) { + store.add(shardCommonStats.store); + } + if (indicesMetrics.contains(IndexMetric.FIELDDATA)) { + fieldData.add(shardCommonStats.fieldData); + } + if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) { + queryCache.add(shardCommonStats.queryCache); + } + if (indicesMetrics.contains(IndexMetric.COMPLETION)) { + completion.add(shardCommonStats.completion); + } + if (indicesMetrics.contains(IndexMetric.SEGMENTS)) { + segments.add(shardCommonStats.segments); } - store.add(shardCommonStats.store); - fieldData.add(shardCommonStats.fieldData); - queryCache.add(shardCommonStats.queryCache); - completion.add(shardCommonStats.completion); - segments.add(shardCommonStats.segments); } } } - shards = new ShardStats(); indexCount = countsPerIndex.size(); - for (final ShardStats indexCountsCursor : countsPerIndex.values()) { - shards.addIndexShardCount(indexCountsCursor); + if (indicesMetrics.contains(IndexMetric.SHARDS)) { + shards = new ShardStats(); + for (final ShardStats indexCountsCursor : countsPerIndex.values()) { + shards.addIndexShardCount(indexCountsCursor); + } } this.mappings = mappingStats; @@ -186,13 +223,27 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.COUNT, indexCount); - shards.toXContent(builder, params); - docs.toXContent(builder, params); - store.toXContent(builder, params); - fieldData.toXContent(builder, params); - queryCache.toXContent(builder, params); - completion.toXContent(builder, params); - segments.toXContent(builder, params); + if (shards != null) { + shards.toXContent(builder, params); + } + if (docs != null) { + docs.toXContent(builder, params); + } + if (store != null) { + store.toXContent(builder, params); + } + if (fieldData != null) { + fieldData.toXContent(builder, params); + } + if (queryCache != null) { + queryCache.toXContent(builder, params); + } + if (completion != null) { + completion.toXContent(builder, params); + } + if (segments != null) { + segments.toXContent(builder, params); + } if (mappings != null) { mappings.toXContent(builder, params); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index b44e9cfc5c74a..bf8218a66fc17 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -36,6 +36,7 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.annotation.PublicApi; @@ -89,10 +90,29 @@ public class ClusterStatsNodes implements ToXContentFragment { private final PackagingTypes packagingTypes; private final IngestStats ingestStats; + public static final Set NODE_STATS_METRICS = Set.of( + // Stats computed from node info and node stat + Metric.OS, + Metric.JVM, + // Stats computed from node stat + Metric.FS, + Metric.PROCESS, + Metric.INGEST, + // Stats computed from node info + Metric.PLUGINS, + Metric.NETWORK_TYPES, + Metric.DISCOVERY_TYPES, + Metric.PACKAGING_TYPES + ); + ClusterStatsNodes(List nodeResponses) { + this(Set.of(Metric.values()), nodeResponses); + } + + ClusterStatsNodes(Set requestedMetrics, List nodeResponses) { this.versions = new HashSet<>(); - this.fs = new FsInfo.Path(); - this.plugins = new HashSet<>(); + this.fs = requestedMetrics.contains(ClusterStatsRequest.Metric.FS) ? new FsInfo.Path() : null; + this.plugins = requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS) ? new HashSet<>() : null; Set seenAddresses = new HashSet<>(nodeResponses.size()); List nodeInfos = new ArrayList<>(nodeResponses.size()); @@ -101,7 +121,9 @@ public class ClusterStatsNodes implements ToXContentFragment { nodeInfos.add(nodeResponse.nodeInfo()); nodeStats.add(nodeResponse.nodeStats()); this.versions.add(nodeResponse.nodeInfo().getVersion()); - this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); + if (requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS)) { + this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos()); + } // now do the stats that should be deduped by hardware (implemented by ip deduping) TransportAddress publishAddress = nodeResponse.nodeInfo().getInfo(TransportInfo.class).address().publishAddress(); @@ -109,18 +131,19 @@ public class ClusterStatsNodes implements ToXContentFragment { if (!seenAddresses.add(inetAddress)) { continue; } - if (nodeResponse.nodeStats().getFs() != null) { + if (requestedMetrics.contains(ClusterStatsRequest.Metric.FS) && nodeResponse.nodeStats().getFs() != null) { this.fs.add(nodeResponse.nodeStats().getFs().getTotal()); } } + this.counts = new Counts(nodeInfos); - this.os = new OsStats(nodeInfos, nodeStats); - this.process = new ProcessStats(nodeStats); - this.jvm = new JvmStats(nodeInfos, nodeStats); - this.networkTypes = new NetworkTypes(nodeInfos); - this.discoveryTypes = new DiscoveryTypes(nodeInfos); - this.packagingTypes = new PackagingTypes(nodeInfos); - this.ingestStats = new IngestStats(nodeStats); + this.networkTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.NETWORK_TYPES) ? new NetworkTypes(nodeInfos) : null; + this.discoveryTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.DISCOVERY_TYPES) ? new DiscoveryTypes(nodeInfos) : null; + this.packagingTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.PACKAGING_TYPES) ? new PackagingTypes(nodeInfos) : null; + this.ingestStats = requestedMetrics.contains(ClusterStatsRequest.Metric.INGEST) ? new IngestStats(nodeStats) : null; + this.process = requestedMetrics.contains(ClusterStatsRequest.Metric.PROCESS) ? new ProcessStats(nodeStats) : null; + this.os = requestedMetrics.contains(ClusterStatsRequest.Metric.OS) ? new OsStats(nodeInfos, nodeStats) : null; + this.jvm = requestedMetrics.contains(ClusterStatsRequest.Metric.JVM) ? new JvmStats(nodeInfos, nodeStats) : null; } public Counts getCounts() { @@ -179,36 +202,54 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); - builder.startObject(Fields.OS); - os.toXContent(builder, params); - builder.endObject(); + if (os != null) { + builder.startObject(Fields.OS); + os.toXContent(builder, params); + builder.endObject(); + } - builder.startObject(Fields.PROCESS); - process.toXContent(builder, params); - builder.endObject(); + if (process != null) { + builder.startObject(Fields.PROCESS); + process.toXContent(builder, params); + builder.endObject(); + } - builder.startObject(Fields.JVM); - jvm.toXContent(builder, params); - builder.endObject(); + if (jvm != null) { + builder.startObject(Fields.JVM); + jvm.toXContent(builder, params); + builder.endObject(); + } - builder.field(Fields.FS); - fs.toXContent(builder, params); + if (fs != null) { + builder.field(Fields.FS); + fs.toXContent(builder, params); + } - builder.startArray(Fields.PLUGINS); - for (PluginInfo pluginInfo : plugins) { - pluginInfo.toXContent(builder, params); + if (plugins != null) { + builder.startArray(Fields.PLUGINS); + for (PluginInfo pluginInfo : plugins) { + pluginInfo.toXContent(builder, params); + } + builder.endArray(); } - builder.endArray(); - builder.startObject(Fields.NETWORK_TYPES); - networkTypes.toXContent(builder, params); - builder.endObject(); + if (networkTypes != null) { + builder.startObject(Fields.NETWORK_TYPES); + networkTypes.toXContent(builder, params); + builder.endObject(); + } - discoveryTypes.toXContent(builder, params); + if (discoveryTypes != null) { + discoveryTypes.toXContent(builder, params); + } - packagingTypes.toXContent(builder, params); + if (packagingTypes != null) { + packagingTypes.toXContent(builder, params); + } - ingestStats.toXContent(builder, params); + if (ingestStats != null) { + ingestStats.toXContent(builder, params); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index b82a9d256a134..1c929881b898b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -39,6 +39,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; /** * A request to get cluster level stats. @@ -48,11 +50,30 @@ @PublicApi(since = "1.0.0") public class ClusterStatsRequest extends BaseNodesRequest { + private final Set requestedMetrics = new HashSet<>(); + private final Set indexMetricsRequested = new HashSet<>(); + private Boolean computeAllMetrics = true; + public ClusterStatsRequest(StreamInput in) throws IOException { super(in); if (in.getVersion().onOrAfter(Version.V_2_16_0)) { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + computeAllMetrics = in.readOptionalBoolean(); + final long longMetricsFlags = in.readLong(); + for (Metric metric : Metric.values()) { + if ((longMetricsFlags & (1 << metric.getIndex())) != 0) { + requestedMetrics.add(metric); + } + } + final long longIndexMetricFlags = in.readLong(); + for (IndexMetric indexMetric : IndexMetric.values()) { + if ((longIndexMetricFlags & (1 << indexMetric.getIndex())) != 0) { + indexMetricsRequested.add(indexMetric); + } + } + } } private Boolean useAggregatedNodeLevelResponses = false; @@ -73,12 +94,133 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon this.useAggregatedNodeLevelResponses = useAggregatedNodeLevelResponses; } + public boolean computeAllMetrics() { + return computeAllMetrics; + } + + public void computeAllMetrics(boolean computeAllMetrics) { + this.computeAllMetrics = computeAllMetrics; + } + + /** + * Add Metric + */ + public ClusterStatsRequest addMetric(Metric metric) { + requestedMetrics.add(metric); + return this; + } + + /** + * Get the names of requested metrics + */ + public Set requestedMetrics() { + return new HashSet<>(requestedMetrics); + } + + /** + * Add IndexMetric + */ + public ClusterStatsRequest addIndexMetric(IndexMetric indexMetric) { + indexMetricsRequested.add(indexMetric); + return this; + } + + public Set indicesMetrics() { + return new HashSet<>(indexMetricsRequested); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_2_16_0)) { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalBoolean(computeAllMetrics); + long longMetricFlags = 0; + for (Metric metric : requestedMetrics) { + longMetricFlags |= (1 << metric.getIndex()); + } + out.writeLong(longMetricFlags); + long longIndexMetricFlags = 0; + for (IndexMetric indexMetric : indexMetricsRequested) { + longIndexMetricFlags |= (1 << indexMetric.getIndex()); + } + out.writeLong(longIndexMetricFlags); + } } + /** + * An enumeration of the "core" sections of metrics that may be requested + * from the cluster stats endpoint. + */ + @PublicApi(since = "3.0.0") + public enum Metric { + OS("os", 0), + JVM("jvm", 1), + FS("fs", 2), + PROCESS("process", 3), + INGEST("ingest", 4), + PLUGINS("plugins", 5), + NETWORK_TYPES("network_types", 6), + DISCOVERY_TYPES("discovery_types", 7), + PACKAGING_TYPES("packaging_types", 8), + INDICES("indices", 9); + + private String metricName; + + private int index; + + Metric(String name, int index) { + this.metricName = name; + this.index = index; + } + + public String metricName() { + return this.metricName; + } + + public int getIndex() { + return index; + } + + } + + /** + * An enumeration of the "core" sections of indices metrics that may be requested + * from the cluster stats endpoint. + * + * When no value is provided for param index_metric, default filter is set to _all. + */ + @PublicApi(since = "3.0.0") + public enum IndexMetric { + // Metrics computed from ShardStats + SHARDS("shards", 0), + DOCS("docs", 1), + STORE("store", 2), + FIELDDATA("fielddata", 3), + QUERY_CACHE("query_cache", 4), + COMPLETION("completion", 5), + SEGMENTS("segments", 6), + // Metrics computed from ClusterState + ANALYSIS("analysis", 7), + MAPPINGS("mappings", 8); + + private String metricName; + + private int index; + + IndexMetric(String name, int index) { + this.metricName = name; + this.index = index; + } + + public String metricName() { + return this.metricName; + } + + public int getIndex() { + return this.index; + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java index 4d0932bd3927d..34fd9ea06235e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequestBuilder.java @@ -36,6 +36,11 @@ import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; +import java.util.Set; + +import static org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import static org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; + /** * Transport request builder for obtaining cluster stats * @@ -55,4 +60,19 @@ public final ClusterStatsRequestBuilder useAggregatedNodeLevelResponses(boolean request.useAggregatedNodeLevelResponses(useAggregatedNodeLevelResponses); return this; } + + public final ClusterStatsRequestBuilder computeAllMetrics(boolean applyMetricFiltering) { + request.computeAllMetrics(applyMetricFiltering); + return this; + } + + public final ClusterStatsRequestBuilder requestMetrics(Set requestMetrics) { + requestMetrics.forEach(request::addMetric); + return this; + } + + public final ClusterStatsRequestBuilder indexMetrics(Set indexMetrics) { + indexMetrics.forEach(request::addIndexMetric); + return this; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java index cc002b689a2a5..870996bb61b23 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -33,6 +33,8 @@ package org.opensearch.action.admin.cluster.stats; import org.opensearch.action.FailedNodeException; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -47,6 +49,7 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Set; /** * Transport response for obtaining cluster stats @@ -88,12 +91,31 @@ public ClusterStatsResponse( List nodes, List failures, ClusterState state + ) { + this(timestamp, clusterUUID, clusterName, nodes, failures, state, Set.of(Metric.values()), Set.of(IndexMetric.values())); + } + + public ClusterStatsResponse( + long timestamp, + String clusterUUID, + ClusterName clusterName, + List nodes, + List failures, + ClusterState state, + Set requestedMetrics, + Set indicesMetrics ) { super(clusterName, nodes, failures); this.clusterUUID = clusterUUID; this.timestamp = timestamp; - nodesStats = new ClusterStatsNodes(nodes); - indicesStats = new ClusterStatsIndices(nodes, MappingStats.of(state), AnalysisStats.of(state)); + nodesStats = requestedMetrics.stream().anyMatch(ClusterStatsNodes.NODE_STATS_METRICS::contains) + ? new ClusterStatsNodes(requestedMetrics, nodes) + : null; + MappingStats mappingStats = indicesMetrics.contains(IndexMetric.MAPPINGS) ? MappingStats.of(state) : null; + AnalysisStats analysisStats = indicesMetrics.contains(IndexMetric.ANALYSIS) ? AnalysisStats.of(state) : null; + indicesStats = requestedMetrics.contains(Metric.INDICES) + ? new ClusterStatsIndices(indicesMetrics, nodes, mappingStats, analysisStats) + : null; ClusterHealthStatus status = null; for (ClusterStatsNodeResponse response : nodes) { // only the cluster-manager node populates the status @@ -131,8 +153,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(timestamp); out.writeOptionalWriteable(status); out.writeOptionalString(clusterUUID); - out.writeOptionalWriteable(indicesStats.getMappings()); - out.writeOptionalWriteable(indicesStats.getAnalysis()); + if (indicesStats != null) { + out.writeOptionalWriteable(indicesStats.getMappings()); + out.writeOptionalWriteable(indicesStats.getAnalysis()); + } else { + out.writeOptionalWriteable(null); + out.writeOptionalWriteable(null); + } } @Override @@ -153,12 +180,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (status != null) { builder.field("status", status.name().toLowerCase(Locale.ROOT)); } - builder.startObject("indices"); - indicesStats.toXContent(builder, params); - builder.endObject(); - builder.startObject("nodes"); - nodesStats.toXContent(builder, params); - builder.endObject(); + if (indicesStats != null) { + builder.startObject("indices"); + indicesStats.toXContent(builder, params); + builder.endObject(); + } + if (nodesStats != null) { + builder.startObject("nodes"); + nodesStats.toXContent(builder, params); + builder.endObject(); + } return builder; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index c4b3524cf6da5..c6581b99eb559 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.ShardStats; @@ -63,7 +64,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; /** * Transport action for obtaining cluster state @@ -76,13 +80,19 @@ public class TransportClusterStatsAction extends TransportNodesAction< TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> { - private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags( + private static final Map SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC = Map.of( CommonStatsFlags.Flag.Docs, + ClusterStatsRequest.IndexMetric.DOCS, CommonStatsFlags.Flag.Store, + ClusterStatsRequest.IndexMetric.STORE, CommonStatsFlags.Flag.FieldData, + ClusterStatsRequest.IndexMetric.FIELDDATA, CommonStatsFlags.Flag.QueryCache, + ClusterStatsRequest.IndexMetric.QUERY_CACHE, CommonStatsFlags.Flag.Completion, - CommonStatsFlags.Flag.Segments + ClusterStatsRequest.IndexMetric.COMPLETION, + CommonStatsFlags.Flag.Segments, + ClusterStatsRequest.IndexMetric.SEGMENTS ); private final NodeService nodeService; @@ -124,14 +134,27 @@ protected ClusterStatsResponse newResponse( + " the cluster state that are too slow for a transport thread" ); ClusterState state = clusterService.state(); - return new ClusterStatsResponse( - System.currentTimeMillis(), - state.metadata().clusterUUID(), - clusterService.getClusterName(), - responses, - failures, - state - ); + if (request.computeAllMetrics()) { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state + ); + } else { + return new ClusterStatsResponse( + System.currentTimeMillis(), + state.metadata().clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + state, + request.requestedMetrics(), + request.indicesMetrics() + ); + } } @Override @@ -149,17 +172,17 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false, false); NodeStats nodeStats = nodeService.stats( CommonStatsFlags.NONE, - true, - true, - true, + isMetricRequired(Metric.OS, nodeRequest.request), + isMetricRequired(Metric.PROCESS, nodeRequest.request), + isMetricRequired(Metric.JVM, nodeRequest.request), false, - true, + isMetricRequired(Metric.FS, nodeRequest.request), false, false, false, false, false, - true, + isMetricRequired(Metric.INGEST, nodeRequest.request), false, false, false, @@ -178,33 +201,36 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false ); List shardsStats = new ArrayList<>(); - for (IndexService indexService : indicesService) { - for (IndexShard indexShard : indexService) { - if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { - // only report on fully started shards - CommitStats commitStats; - SeqNoStats seqNoStats; - RetentionLeaseStats retentionLeaseStats; - try { - commitStats = indexShard.commitStats(); - seqNoStats = indexShard.seqNoStats(); - retentionLeaseStats = indexShard.getRetentionLeaseStats(); - } catch (final AlreadyClosedException e) { - // shard is closed - no stats is fine - commitStats = null; - seqNoStats = null; - retentionLeaseStats = null; + if (isMetricRequired(Metric.INDICES, nodeRequest.request)) { + CommonStatsFlags commonStatsFlags = getCommonStatsFlags(nodeRequest); + for (IndexService indexService : indicesService) { + for (IndexShard indexShard : indexService) { + if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { + // only report on fully started shards + CommitStats commitStats; + SeqNoStats seqNoStats; + RetentionLeaseStats retentionLeaseStats; + try { + commitStats = indexShard.commitStats(); + seqNoStats = indexShard.seqNoStats(); + retentionLeaseStats = indexShard.getRetentionLeaseStats(); + } catch (final AlreadyClosedException e) { + // shard is closed - no stats is fine + commitStats = null; + seqNoStats = null; + retentionLeaseStats = null; + } + shardsStats.add( + new ShardStats( + indexShard.routingEntry(), + indexShard.shardPath(), + new CommonStats(indicesService.getIndicesQueryCache(), indexShard, commonStatsFlags), + commitStats, + seqNoStats, + retentionLeaseStats + ) + ); } - shardsStats.add( - new ShardStats( - indexShard.routingEntry(), - indexShard.shardPath(), - new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), - commitStats, - seqNoStats, - retentionLeaseStats - ) - ); } } } @@ -224,6 +250,31 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq ); } + /** + * A metric is required when: all cluster stats are required (OR) if the metric is requested + * @param metric + * @param clusterStatsRequest + * @return + */ + private boolean isMetricRequired(Metric metric, ClusterStatsRequest clusterStatsRequest) { + return clusterStatsRequest.computeAllMetrics() || clusterStatsRequest.requestedMetrics().contains(metric); + } + + private static CommonStatsFlags getCommonStatsFlags(ClusterStatsNodeRequest nodeRequest) { + Set requestedCommonStatsFlags = new HashSet<>(); + if (nodeRequest.request.computeAllMetrics()) { + requestedCommonStatsFlags.addAll(SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC.keySet()); + } else { + for (Map.Entry entry : SHARDS_STATS_FLAG_MAP_TO_INDEX_METRIC + .entrySet()) { + if (nodeRequest.request.indicesMetrics().contains(entry.getValue())) { + requestedCommonStatsFlags.add(entry.getKey()); + } + } + } + return new CommonStatsFlags(requestedCommonStatsFlags.toArray(new CommonStatsFlags.Flag[0])); + } + /** * Inner Cluster Stats Node Request * diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java index d844e5cbb8897..8ca3157d9f775 100644 --- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java @@ -77,6 +77,9 @@ public QueryCacheStats(long ramBytesUsed, long hitCount, long missCount, long ca } public void add(QueryCacheStats stats) { + if (stats == null) { + return; + } ramBytesUsed += stats.ramBytesUsed; hitCount += stats.hitCount; missCount += stats.missCount; diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java index 85b435e969bfa..1bd81c840a3d4 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java @@ -80,6 +80,9 @@ public FieldDataStats(long memorySize, long evictions, @Nullable FieldMemoryStat } public void add(FieldDataStats stats) { + if (stats == null) { + return; + } this.memorySize += stats.memorySize; this.evictions += stats.evictions; if (stats.fields != null) { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java index ee33bd18db05d..47f3e048c516a 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -33,13 +33,24 @@ package org.opensearch.rest.action.admin.cluster; import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric; import org.opensearch.client.node.NodeClient; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions.NodesResponseRestListener; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -54,7 +65,34 @@ public class RestClusterStatsAction extends BaseRestHandler { @Override public List routes() { - return unmodifiableList(asList(new Route(GET, "/_cluster/stats"), new Route(GET, "/_cluster/stats/nodes/{nodeId}"))); + return unmodifiableList( + asList( + new Route(GET, "/_cluster/stats"), + new Route(GET, "/_cluster/stats/nodes/{nodeId}"), + new Route(GET, "/_cluster/stats/{metric}/nodes/{nodeId}"), + new Route(GET, "/_cluster/stats/{metric}/{index_metric}/nodes/{nodeId}") + ) + ); + } + + static final Map> INDEX_METRIC_TO_REQUEST_CONSUMER_MAP; + + static final Map> METRIC_REQUEST_CONSUMER_MAP; + + static { + Map> metricRequestConsumerMap = new HashMap<>(); + for (Metric metric : Metric.values()) { + metricRequestConsumerMap.put(metric.metricName(), request -> request.addMetric(metric)); + } + METRIC_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricRequestConsumerMap); + } + + static { + Map> metricMap = new HashMap<>(); + for (IndexMetric indexMetric : IndexMetric.values()) { + metricMap.put(indexMetric.metricName(), request -> request.addIndexMetric(indexMetric)); + } + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP = Collections.unmodifiableMap(metricMap); } @Override @@ -64,10 +102,101 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); + ClusterStatsRequest clusterStatsRequest = fromRequest(request); + return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); + } + + public static ClusterStatsRequest fromRequest(final RestRequest request) { + Set metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all")); + // Value for param index_metric defaults to _all when indices metric or all metrics are requested. + String indicesMetricsDefaultValue = metrics.contains(Metric.INDICES.metricName()) || metrics.contains("_all") ? "_all" : null; + Set indexMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", indicesMetricsDefaultValue)); + String[] nodeIds = request.paramAsStringArray("nodeId", null); + + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(nodeIds); clusterStatsRequest.timeout(request.param("timeout")); clusterStatsRequest.useAggregatedNodeLevelResponses(true); - return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); + clusterStatsRequest.computeAllMetrics(false); + + paramValidations(metrics, indexMetrics, request); + final Set metricsRequested = metrics.contains("_all") + ? new HashSet<>(METRIC_REQUEST_CONSUMER_MAP.keySet()) + : new HashSet<>(metrics); + Set invalidMetrics = validateAndSetRequestedMetrics(metricsRequested, METRIC_REQUEST_CONSUMER_MAP, clusterStatsRequest); + if (!invalidMetrics.isEmpty()) { + throw new IllegalArgumentException( + unrecognizedStrings(request, invalidMetrics, METRIC_REQUEST_CONSUMER_MAP.keySet(), "metric") + ); + } + if (metricsRequested.contains(Metric.INDICES.metricName())) { + final Set indexMetricsRequested = indexMetrics.contains("_all") + ? INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet() + : new HashSet<>(indexMetrics); + Set invalidIndexMetrics = validateAndSetRequestedMetrics( + indexMetricsRequested, + INDEX_METRIC_TO_REQUEST_CONSUMER_MAP, + clusterStatsRequest + ); + if (!invalidIndexMetrics.isEmpty()) { + throw new IllegalArgumentException( + unrecognizedStrings(request, invalidIndexMetrics, INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet(), "index metric") + ); + } + } + + return clusterStatsRequest; + } + + private static void paramValidations(Set metrics, Set indexMetrics, RestRequest request) { + if (metrics.size() > 1 && metrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ); + } + + if (indexMetrics.size() > 1 && indexMetrics.contains("_all")) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual index metrics [%s]", + request.path(), + request.param("index_metric") + ) + ); + } + + if (!metrics.contains(Metric.INDICES.metricName()) && !metrics.contains("_all") && !indexMetrics.isEmpty()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains index metrics [%s] but indices stats not requested", + request.path(), + request.param("index_metric") + ) + ); + } + } + + private static Set validateAndSetRequestedMetrics( + Set metrics, + Map> metricConsumerMap, + ClusterStatsRequest clusterStatsRequest + ) { + final Set invalidMetrics = new TreeSet<>(); + for (String metric : metrics) { + Consumer clusterStatsRequestConsumer = metricConsumerMap.get(metric); + if (clusterStatsRequestConsumer != null) { + clusterStatsRequestConsumer.accept(clusterStatsRequest); + } else { + invalidMetrics.add(metric); + } + } + return invalidMetrics; } @Override diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java new file mode 100644 index 0000000000000..ad7706292d93c --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java @@ -0,0 +1,281 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.stats; + +import org.opensearch.Build; +import org.opensearch.Version; +import org.opensearch.action.admin.cluster.node.info.NodeInfo; +import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric; +import org.opensearch.action.admin.indices.stats.CommonStats; +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.ShardStats; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.index.Index; +import org.opensearch.index.cache.query.QueryCacheStats; +import org.opensearch.index.engine.SegmentsStats; +import org.opensearch.index.fielddata.FieldDataStats; +import org.opensearch.index.flush.FlushStats; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.shard.IndexingStats; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.StoreStats; +import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.monitor.jvm.JvmStats; +import org.opensearch.monitor.os.OsInfo; +import org.opensearch.monitor.process.ProcessStats; +import org.opensearch.search.suggest.completion.CompletionStats; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.TransportInfo; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class ClusterStatsResponseTests extends OpenSearchTestCase { + + public void testSerializationWithIndicesMappingAndAnalysisStats() throws Exception { + List defaultClusterStatsNodeResponses = new ArrayList<>(); + + int numberOfNodes = randomIntBetween(1, 4); + Index testIndex = new Index("test-index", "_na_"); + + for (int i = 0; i < numberOfNodes; i++) { + DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createShardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats); + defaultClusterStatsNodeResponses.add(customClusterStatsResponse); + } + ClusterStatsResponse clusterStatsResponse = new ClusterStatsResponse( + 1l, + "UUID", + new ClusterName("cluster_name"), + defaultClusterStatsNodeResponses, + List.of(), + ClusterState.EMPTY_STATE, + Set.of(ClusterStatsRequest.Metric.INDICES), + Set.of(IndexMetric.MAPPINGS, IndexMetric.ANALYSIS) + ); + BytesStreamOutput output = new BytesStreamOutput(); + clusterStatsResponse.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + ClusterStatsResponse deserializedClusterStatsResponse = new ClusterStatsResponse(streamInput); + assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); + assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); + assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); + assertNotNull(clusterStatsResponse.indicesStats); + assertEquals(clusterStatsResponse.indicesStats.getMappings(), deserializedClusterStatsResponse.indicesStats.getMappings()); + assertEquals(clusterStatsResponse.indicesStats.getAnalysis(), deserializedClusterStatsResponse.indicesStats.getAnalysis()); + } + + public void testSerializationWithoutIndicesMappingAndAnalysisStats() throws Exception { + List defaultClusterStatsNodeResponses = new ArrayList<>(); + + int numberOfNodes = randomIntBetween(1, 4); + Index testIndex = new Index("test-index", "_na_"); + + for (int i = 0; i < numberOfNodes; i++) { + DiscoveryNode node = new DiscoveryNode("node-" + i, buildNewFakeTransportAddress(), Version.CURRENT); + CommonStats commonStats = createRandomCommonStats(); + ShardStats[] shardStats = createShardStats(node, testIndex, commonStats); + ClusterStatsNodeResponse customClusterStatsResponse = createClusterStatsNodeResponse(node, shardStats); + defaultClusterStatsNodeResponses.add(customClusterStatsResponse); + } + ClusterStatsResponse clusterStatsResponse = new ClusterStatsResponse( + 1l, + "UUID", + new ClusterName("cluster_name"), + defaultClusterStatsNodeResponses, + List.of(), + ClusterState.EMPTY_STATE, + Set.of(ClusterStatsRequest.Metric.INDICES, ClusterStatsRequest.Metric.PROCESS, ClusterStatsRequest.Metric.JVM), + Set.of( + IndexMetric.DOCS, + IndexMetric.STORE, + IndexMetric.SEGMENTS, + IndexMetric.QUERY_CACHE, + IndexMetric.FIELDDATA, + IndexMetric.COMPLETION + ) + ); + BytesStreamOutput output = new BytesStreamOutput(); + clusterStatsResponse.writeTo(output); + + StreamInput streamInput = output.bytes().streamInput(); + ClusterStatsResponse deserializedClusterStatsResponse = new ClusterStatsResponse(streamInput); + assertEquals(clusterStatsResponse.timestamp, deserializedClusterStatsResponse.timestamp); + assertEquals(clusterStatsResponse.status, deserializedClusterStatsResponse.status); + assertEquals(clusterStatsResponse.clusterUUID, deserializedClusterStatsResponse.clusterUUID); + assertNotNull(deserializedClusterStatsResponse.nodesStats); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getProcess()); + assertNotNull(deserializedClusterStatsResponse.nodesStats.getJvm()); + assertNotNull(deserializedClusterStatsResponse.indicesStats); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getDocs()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getStore()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getSegments()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getQueryCache()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getFieldData()); + assertNotNull(deserializedClusterStatsResponse.indicesStats.getCompletion()); + assertNull(deserializedClusterStatsResponse.indicesStats.getMappings()); + assertNull(deserializedClusterStatsResponse.indicesStats.getAnalysis()); + } + + private ClusterStatsNodeResponse createClusterStatsNodeResponse(DiscoveryNode node, ShardStats[] shardStats) throws IOException { + JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[1]; + garbageCollectorsArray[0] = new JvmStats.GarbageCollector( + randomAlphaOfLengthBetween(3, 10), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray); + NodeInfo nodeInfo = new NodeInfo( + Version.CURRENT, + Build.CURRENT, + node, + Settings.EMPTY, + new OsInfo(randomLong(), randomInt(), randomInt(), "name", "pretty_name", "arch", "version"), + null, + JvmInfo.jvmInfo(), + null, + new TransportInfo( + new BoundTransportAddress(new TransportAddress[] { buildNewFakeTransportAddress() }, buildNewFakeTransportAddress()), + null + ), + null, + new PluginsAndModules(Collections.emptyList(), Collections.emptyList()), + null, + null, + null, + null + ); + + NodeStats nodeStats = new NodeStats( + node, + randomNonNegativeLong(), + null, + null, + new ProcessStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()), + new ProcessStats.Mem(randomNonNegativeLong()) + ), + new JvmStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + new JvmStats.Mem( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + Collections.emptyList() + ), + new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), + garbageCollectors, + Collections.emptyList(), + new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) + ), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + return new ClusterStatsNodeResponse(node, null, nodeInfo, nodeStats, shardStats); + + } + + private CommonStats createRandomCommonStats() { + CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE); + commonStats.docs = new DocsStats(randomLongBetween(0, 10000), randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.store = new StoreStats(randomLongBetween(0, 100), randomLongBetween(0, 1000)); + commonStats.indexing = new IndexingStats(); + commonStats.completion = new CompletionStats(); + commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100)); + commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null); + commonStats.queryCache = new QueryCacheStats( + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100), + randomLongBetween(0, 100) + ); + commonStats.segments = new SegmentsStats(); + + return commonStats; + } + + private ShardStats[] createShardStats(DiscoveryNode localNode, Index index, CommonStats commonStats) { + List shardStatsList = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + index.getName(), + i, + localNode.getId(), + randomBoolean(), + shardRoutingState + ); + + Path path = createTempDir().resolve("indices") + .resolve(shardRouting.shardId().getIndex().getUUID()) + .resolve(String.valueOf(shardRouting.shardId().id())); + + ShardStats shardStats = new ShardStats( + shardRouting, + new ShardPath(false, path, path, shardRouting.shardId()), + commonStats, + null, + null, + null + ); + shardStatsList.add(shardStats); + } + + return shardStatsList.toArray(new ShardStats[0]); + } + +} diff --git a/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java new file mode 100644 index 0000000000000..8b46f676636fd --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/action/admin/cluster/RestClusterStatsActionTests.java @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.cluster; + +import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; + +import java.util.HashMap; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.object.HasToString.hasToString; +import static org.mockito.Mockito.mock; + +public class RestClusterStatsActionTests extends OpenSearchTestCase { + + private RestClusterStatsAction action; + + @Override + public void setUp() throws Exception { + super.setUp(); + action = new RestClusterStatsAction(); + } + + public void testFromRequestBasePath() { + final HashMap params = new HashMap<>(); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertFalse(clusterStatsRequest.computeAllMetrics()); + assertNotNull(clusterStatsRequest.requestedMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + for (ClusterStatsRequest.Metric metric : ClusterStatsRequest.Metric.values()) { + assertTrue(clusterStatsRequest.requestedMetrics().contains(metric)); + } + assertNotNull(clusterStatsRequest.indicesMetrics()); + assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); + for (ClusterStatsRequest.IndexMetric indexMetric : ClusterStatsRequest.IndexMetric.values()) { + assertTrue(clusterStatsRequest.indicesMetrics().contains(indexMetric)); + } + } + + public void testFromRequestWithNodeStatsMetricsFilter() { + Set metricsRequested = Set.of(ClusterStatsRequest.Metric.OS, ClusterStatsRequest.Metric.FS); + String metricParam = metricsRequested.stream().map(ClusterStatsRequest.Metric::metricName).collect(Collectors.joining(",")); + final HashMap params = new HashMap<>(); + params.put("metric", metricParam); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertFalse(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + assertEquals(2, clusterStatsRequest.requestedMetrics().size()); + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); + assertTrue(clusterStatsRequest.indicesMetrics().isEmpty()); + } + + public void testFromRequestWithIndicesStatsMetricsFilter() { + Set metricsRequested = Set.of( + ClusterStatsRequest.Metric.OS, + ClusterStatsRequest.Metric.FS, + ClusterStatsRequest.Metric.INDICES + ); + Set indicesMetricsRequested = Set.of( + ClusterStatsRequest.IndexMetric.SHARDS, + ClusterStatsRequest.IndexMetric.SEGMENTS + ); + String metricParam = metricsRequested.stream().map(ClusterStatsRequest.Metric::metricName).collect(Collectors.joining(",")); + String indicesMetricParam = indicesMetricsRequested.stream() + .map(ClusterStatsRequest.IndexMetric::metricName) + .collect(Collectors.joining(",")); + final HashMap params = new HashMap<>(); + params.put("metric", metricParam); + params.put("index_metric", indicesMetricParam); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + ClusterStatsRequest clusterStatsRequest = RestClusterStatsAction.fromRequest(request); + assertNotNull(clusterStatsRequest); + assertTrue(clusterStatsRequest.useAggregatedNodeLevelResponses()); + assertFalse(clusterStatsRequest.computeAllMetrics()); + assertFalse(clusterStatsRequest.requestedMetrics().isEmpty()); + assertEquals(3, clusterStatsRequest.requestedMetrics().size()); + assertEquals(metricsRequested, clusterStatsRequest.requestedMetrics()); + assertFalse(clusterStatsRequest.indicesMetrics().isEmpty()); + assertEquals(2, clusterStatsRequest.indicesMetrics().size()); + assertEquals(indicesMetricsRequested, clusterStatsRequest.indicesMetrics()); + } + + public void testUnrecognizedMetric() { + final HashMap params = new HashMap<>(); + final String metric = randomAlphaOfLength(64); + params.put("metric", metric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains unrecognized metric: [" + metric + "]"))); + } + + public void testUnrecognizedIndexMetric() { + final HashMap params = new HashMap<>(); + params.put("metric", "_all,"); + final String indexMetric = randomAlphaOfLength(64); + params.put("index_metric", indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains unrecognized index metric: [" + indexMetric + "]"))); + } + + public void testAllMetricsRequestWithOtherMetric() { + final HashMap params = new HashMap<>(); + final String metric = randomSubsetOf(1, RestClusterStatsAction.METRIC_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("metric", "_all," + metric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat(e, hasToString(containsString("request [/_cluster/stats] contains _all and individual metrics [_all," + metric + "]"))); + } + + public void testAllIndexMetricsRequestWithOtherIndicesMetric() { + final HashMap params = new HashMap<>(); + params.put("metric", "_all,"); + final String indexMetric = randomSubsetOf(1, RestClusterStatsAction.INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("index_metric", "_all," + indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat( + e, + hasToString(containsString("request [/_cluster/stats] contains _all and individual index metrics [_all," + indexMetric + "]")) + ); + } + + public void testIndexMetricsRequestWithoutMetricIndices() { + final HashMap params = new HashMap<>(); + params.put("metric", "os"); + final String indexMetric = randomSubsetOf(1, RestClusterStatsAction.INDEX_METRIC_TO_REQUEST_CONSUMER_MAP.keySet()).get(0); + params.put("index_metric", indexMetric); + final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_cluster/stats").withParams(params).build(); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> action.prepareRequest(request, mock(NodeClient.class)) + ); + assertThat( + e, + hasToString( + containsString("request [/_cluster/stats] contains index metrics [" + indexMetric + "] but indices stats not requested") + ) + ); + } + +} From 20e233e3b432957fc46b9f1b6953c8b556c87997 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Tue, 22 Oct 2024 22:28:58 +0530 Subject: [PATCH 270/357] Fixing inline javadocs usage in PaginationStrategy (#16428) Signed-off-by: Harsh Garg Co-authored-by: Harsh Garg --- .../action/pagination/PaginationStrategy.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java index edb2489b1e4f0..8cc1ebdd2f194 100644 --- a/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/action/pagination/PaginationStrategy.java @@ -44,8 +44,11 @@ public interface PaginationStrategy { List getRequestedEntities(); /** - * - * Utility method to get list of indices filtered as per {@param filterPredicate} and the sorted according to {@param comparator}. + * Utility method to get list of indices filtered and sorted as per the provided parameters. + * @param clusterState state consisting of all the indices to be filtered and sorted. + * @param filterPredicate predicate to be used for filtering out the required indices. + * @param comparator comparator to be used for sorting the already filtered list of indices. + * @return list of filtered and sorted IndexMetadata. */ static List getSortedIndexMetadata( final ClusterState clusterState, @@ -56,8 +59,10 @@ static List getSortedIndexMetadata( } /** - * - * Utility method to get list of indices sorted as per {@param comparator}. + * Utility method to get list of sorted indices. + * @param clusterState state consisting of indices to be sorted. + * @param comparator comparator to be used for sorting the list of indices. + * @return list of sorted IndexMetadata. */ static List getSortedIndexMetadata(final ClusterState clusterState, Comparator comparator) { return clusterState.metadata().indices().values().stream().sorted(comparator).collect(Collectors.toList()); From 267c68ef9d4e7df8559693350b4c013f493f402e Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 22 Oct 2024 10:52:34 -0700 Subject: [PATCH 271/357] Fix unclosed store references with node-node segrep when primary node is unknown. (#16106) This PR fixes a bug with node-node pull based replication where if the replica does not know the DiscoveryNode of its primary we would fail after constructing a SegmentReplicationTarget that holds a store reference. Only after replication is started would a failure occur because the source node is null, and the target would not get cleaned up. Push based replication already handled this case by catching any error and closing the target. This update ensures the validation is done before constructing our PrimaryShardReplicationSource, before any target object is created in both cases push and pull. Signed-off-by: Marc Handalian --- .../indices/settings/SearchOnlyReplicaIT.java | 2 + .../PrimaryShardReplicationSource.java | 2 + .../SegmentReplicationSourceFactory.java | 6 ++- .../replication/SegmentReplicatorTests.java | 50 +++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java index 6bd91df1de66f..fa836e2cc5784 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java @@ -126,6 +126,8 @@ public void testFailoverWithSearchReplica_WithoutWriterReplicas() throws IOExcep .put(indexSettings()) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numWriterReplicas) .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, numSearchReplicas) + .put("index.refresh_interval", "40ms") // set lower interval so replica attempts replication cycles after primary is + // removed. .build() ); ensureYellow(TEST_INDEX); diff --git a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java index a17779810239a..af37594f88fee 100644 --- a/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java +++ b/server/src/main/java/org/opensearch/indices/replication/PrimaryShardReplicationSource.java @@ -47,6 +47,8 @@ public PrimaryShardReplicationSource( RecoverySettings recoverySettings, DiscoveryNode sourceNode ) { + assert targetNode != null : "Target node must be set"; + assert sourceNode != null : "Source node must be set"; this.targetAllocationId = targetAllocationId; this.transportService = transportService; this.sourceNode = sourceNode; diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java index 81eb38757aebe..2b6723512abf2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java @@ -53,6 +53,10 @@ public SegmentReplicationSource get(IndexShard shard) { private DiscoveryNode getPrimaryNode(ShardId shardId) { ShardRouting primaryShard = clusterService.state().routingTable().shardRoutingTable(shardId).primaryShard(); - return clusterService.state().nodes().get(primaryShard.currentNodeId()); + DiscoveryNode node = clusterService.state().nodes().get(primaryShard.currentNodeId()); + if (node == null) { + throw new IllegalStateException("Cannot replicate, primary shard for " + shardId + " is not allocated on any node"); + } + return node; } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java index 7acee449a1b46..81ea16c80dd79 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicatorTests.java @@ -10,7 +10,13 @@ import org.apache.lucene.store.IOContext; import org.opensearch.OpenSearchCorruptionException; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.UnassignedInfo; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; @@ -19,10 +25,12 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.CopyState; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; import java.io.IOException; import java.io.UncheckedIOException; @@ -45,6 +53,48 @@ public class SegmentReplicatorTests extends IndexShardTestCase { .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); + public void testReplicationWithUnassignedPrimary() throws Exception { + final IndexShard replica = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + final IndexShard primary = newStartedShard(true, settings, new NRTReplicationEngineFactory()); + SegmentReplicator replicator = new SegmentReplicator(threadPool); + + ClusterService cs = mock(ClusterService.class); + IndexShardRoutingTable.Builder shardRoutingTable = new IndexShardRoutingTable.Builder(replica.shardId()); + shardRoutingTable.addShard(replica.routingEntry()); + shardRoutingTable.addShard(primary.routingEntry().moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test"))); + + when(cs.state()).thenReturn(buildClusterState(replica, shardRoutingTable)); + replicator.setSourceFactory(new SegmentReplicationSourceFactory(mock(TransportService.class), mock(RecoverySettings.class), cs)); + expectThrows(IllegalStateException.class, () -> replicator.startReplication(replica)); + closeShards(replica, primary); + } + + public void testReplicationWithUnknownPrimaryNode() throws Exception { + final IndexShard replica = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + final IndexShard primary = newStartedShard(true, settings, new NRTReplicationEngineFactory()); + SegmentReplicator replicator = new SegmentReplicator(threadPool); + + ClusterService cs = mock(ClusterService.class); + IndexShardRoutingTable.Builder shardRoutingTable = new IndexShardRoutingTable.Builder(replica.shardId()); + shardRoutingTable.addShard(replica.routingEntry()); + shardRoutingTable.addShard(primary.routingEntry()); + + when(cs.state()).thenReturn(buildClusterState(replica, shardRoutingTable)); + replicator.setSourceFactory(new SegmentReplicationSourceFactory(mock(TransportService.class), mock(RecoverySettings.class), cs)); + expectThrows(IllegalStateException.class, () -> replicator.startReplication(replica)); + closeShards(replica, primary); + } + + private ClusterState buildClusterState(IndexShard replica, IndexShardRoutingTable.Builder indexShard) { + return ClusterState.builder(clusterService.state()) + .routingTable( + RoutingTable.builder() + .add(IndexRoutingTable.builder(replica.shardId().getIndex()).addIndexShard(indexShard.build()).build()) + .build() + ) + .build(); + } + public void testStartReplicationWithoutSourceFactory() { ThreadPool threadpool = mock(ThreadPool.class); ExecutorService mock = mock(ExecutorService.class); From 5120efbcd8bfceeb236421ddbe4c01f1069a1850 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 22 Oct 2024 14:57:37 -0400 Subject: [PATCH 272/357] Update JDK to 23.0.1 (#16429) Signed-off-by: Andriy Redko --- .../java/org/opensearch/gradle/test/DistroTestPlugin.java | 4 ++-- buildSrc/version.properties | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 9365f1c732229..439d0de39584d 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -77,9 +77,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "23+37"; + private static final String SYSTEM_JDK_VERSION = "23.0.1+11"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "23+37"; + private static final String GRADLE_JDK_VERSION = "23.0.1+11"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 5740c124910b9..f9a8bee5783b1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -2,7 +2,7 @@ opensearch = 3.0.0 lucene = 9.12.0 bundled_jdk_vendor = adoptium -bundled_jdk = 23+37 +bundled_jdk = 23.0.1+11 # optional dependencies spatial4j = 0.7 From 6891267608913dc27b21a65156ddd8817a8e9746 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 22 Oct 2024 15:05:40 -0700 Subject: [PATCH 273/357] Improve the rejection logic for soft mode query groups during node duress (#16417) * improve the rejection logic for wlm Signed-off-by: Kaushal Kumar * add CHANGELOG Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kaushal Kumar --- CHANGELOG.md | 3 +- .../org/opensearch/wlm/QueryGroupService.java | 7 +- .../wlm/QueryGroupServiceTests.java | 81 +++++++++++++++---- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 832871453028b..8eddd2c750677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,7 +87,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) -- [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) +- [Workload Management] Make query groups persistent across process restarts ([#16370](https://github.com/opensearch-project/OpenSearch/pull/16370)) +- [Workload Management] Enhance rejection mechanism in workload management ([#16417](https://github.com/opensearch-project/OpenSearch/pull/16417)) - Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) - Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) - Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java index cb0be5597766a..14002a2b38134 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupService.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupService.java @@ -266,11 +266,12 @@ public void rejectIfNeeded(String queryGroupId) { return; } - // rejections will not happen for SOFT mode QueryGroups + // rejections will not happen for SOFT mode QueryGroups unless node is in duress Optional optionalQueryGroup = activeQueryGroups.stream().filter(x -> x.get_id().equals(queryGroupId)).findFirst(); - if (optionalQueryGroup.isPresent() && optionalQueryGroup.get().getResiliencyMode() == MutableQueryGroupFragment.ResiliencyMode.SOFT) - return; + if (optionalQueryGroup.isPresent() + && (optionalQueryGroup.get().getResiliencyMode() == MutableQueryGroupFragment.ResiliencyMode.SOFT + && !nodeDuressTrackers.isNodeInDuress())) return; optionalQueryGroup.ifPresent(queryGroup -> { boolean reject = false; diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java index 45428865259c3..f22759ce968aa 100644 --- a/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupServiceTests.java @@ -48,6 +48,7 @@ import static org.mockito.Mockito.when; public class QueryGroupServiceTests extends OpenSearchTestCase { + public static final String QUERY_GROUP_ID = "queryGroupId1"; private QueryGroupService queryGroupService; private QueryGroupTaskCancellationService mockCancellationService; private ClusterService mockClusterService; @@ -68,6 +69,7 @@ public void setUp() throws Exception { mockNodeDuressTrackers = Mockito.mock(NodeDuressTrackers.class); mockCancellationService = Mockito.mock(TestQueryGroupCancellationService.class); mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(); + when(mockNodeDuressTrackers.isNodeInDuress()).thenReturn(false); queryGroupService = new QueryGroupService( mockCancellationService, @@ -203,26 +205,52 @@ public void testRejectIfNeeded_whenQueryGroupIdIsNullOrDefaultOne() { verify(spyMap, never()).get(any()); } + public void testRejectIfNeeded_whenSoftModeQueryGroupIsContendedAndNodeInDuress() { + Set activeQueryGroups = getActiveQueryGroups( + "testQueryGroup", + QUERY_GROUP_ID, + MutableQueryGroupFragment.ResiliencyMode.SOFT, + Map.of(ResourceType.CPU, 0.10) + ); + mockQueryGroupStateMap = new HashMap<>(); + mockQueryGroupStateMap.put("queryGroupId1", new QueryGroupState()); + QueryGroupState state = new QueryGroupState(); + QueryGroupState.ResourceTypeState cpuResourceState = new QueryGroupState.ResourceTypeState(ResourceType.CPU); + cpuResourceState.setLastRecordedUsage(0.10); + state.getResourceState().put(ResourceType.CPU, cpuResourceState); + QueryGroupState spyState = spy(state); + mockQueryGroupStateMap.put(QUERY_GROUP_ID, spyState); + + mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); + + queryGroupService = new QueryGroupService( + mockCancellationService, + mockClusterService, + mockThreadPool, + mockWorkloadManagementSettings, + mockNodeDuressTrackers, + mockQueryGroupsStateAccessor, + activeQueryGroups, + new HashSet<>() + ); + when(mockWorkloadManagementSettings.getWlmMode()).thenReturn(WlmMode.ENABLED); + when(mockNodeDuressTrackers.isNodeInDuress()).thenReturn(true); + assertThrows(OpenSearchRejectedExecutionException.class, () -> queryGroupService.rejectIfNeeded("queryGroupId1")); + } + public void testRejectIfNeeded_whenQueryGroupIsSoftMode() { - QueryGroup testQueryGroup = new QueryGroup( + Set activeQueryGroups = getActiveQueryGroups( "testQueryGroup", - "queryGroupId1", - new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.SOFT, Map.of(ResourceType.CPU, 0.10)), - 1L + QUERY_GROUP_ID, + MutableQueryGroupFragment.ResiliencyMode.SOFT, + Map.of(ResourceType.CPU, 0.10) ); - Set activeQueryGroups = new HashSet<>() { - { - add(testQueryGroup); - } - }; mockQueryGroupStateMap = new HashMap<>(); QueryGroupState spyState = spy(new QueryGroupState()); mockQueryGroupStateMap.put("queryGroupId1", spyState); mockQueryGroupsStateAccessor = new QueryGroupsStateAccessor(mockQueryGroupStateMap); - Map spyMap = spy(mockQueryGroupStateMap); - queryGroupService = new QueryGroupService( mockCancellationService, mockClusterService, @@ -239,11 +267,11 @@ public void testRejectIfNeeded_whenQueryGroupIsSoftMode() { } public void testRejectIfNeeded_whenQueryGroupIsEnforcedMode_andNotBreaching() { - QueryGroup testQueryGroup = new QueryGroup( + QueryGroup testQueryGroup = getQueryGroup( "testQueryGroup", "queryGroupId1", - new MutableQueryGroupFragment(MutableQueryGroupFragment.ResiliencyMode.ENFORCED, Map.of(ResourceType.CPU, 0.10)), - 1L + MutableQueryGroupFragment.ResiliencyMode.ENFORCED, + Map.of(ResourceType.CPU, 0.10) ); QueryGroup spuQueryGroup = spy(testQueryGroup); Set activeQueryGroups = new HashSet<>() { @@ -464,6 +492,31 @@ public void testShouldSBPHandle() { } + private static Set getActiveQueryGroups( + String name, + String id, + MutableQueryGroupFragment.ResiliencyMode mode, + Map resourceLimits + ) { + QueryGroup testQueryGroup = getQueryGroup(name, id, mode, resourceLimits); + Set activeQueryGroups = new HashSet<>() { + { + add(testQueryGroup); + } + }; + return activeQueryGroups; + } + + private static QueryGroup getQueryGroup( + String name, + String id, + MutableQueryGroupFragment.ResiliencyMode mode, + Map resourceLimits + ) { + QueryGroup testQueryGroup = new QueryGroup(name, id, new MutableQueryGroupFragment(mode, resourceLimits), 1L); + return testQueryGroup; + } + // This is needed to test the behavior of QueryGroupService#doRun method static class TestQueryGroupCancellationService extends QueryGroupTaskCancellationService { public TestQueryGroupCancellationService( From 760e67641a063adfec05721ef676e47d347b17a5 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 22 Oct 2024 17:47:29 -0700 Subject: [PATCH 274/357] Wlm create/update REST API bug fix (#16422) * test changes Signed-off-by: Kaushal Kumar * fix the create/update queryGroup REST APIs Signed-off-by: Kaushal Kumar * undo gradle change Signed-off-by: Kaushal Kumar * add PR link in CHANGELOG Signed-off-by: Kaushal Kumar * fix javadoc issues Signed-off-by: Kaushal Kumar * remove redundant name param Signed-off-by: Kaushal Kumar * Update CHANGELOG.md Signed-off-by: Ankit Jain * fix action name in transport class for update query group Signed-off-by: Kaushal Kumar --------- Signed-off-by: Kaushal Kumar Signed-off-by: Ankit Jain Co-authored-by: Ankit Jain --- CHANGELOG.md | 1 + .../wlm/action/CreateQueryGroupRequest.java | 4 +- .../TransportCreateQueryGroupAction.java | 53 ++++++++++++++++--- .../TransportUpdateQueryGroupAction.java | 52 +++++++++++++++--- .../wlm/action/UpdateQueryGroupRequest.java | 4 +- 5 files changed, 96 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eddd2c750677..c1485de3de2ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) - Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) - Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) +- [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422)) ### Security diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java index d92283391dd3b..1ce04faa7ccc1 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java @@ -8,8 +8,8 @@ package org.opensearch.plugin.wlm.action; -import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.metadata.QueryGroup; import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.StreamInput; @@ -33,7 +33,7 @@ * * @opensearch.experimental */ -public class CreateQueryGroupRequest extends ActionRequest { +public class CreateQueryGroupRequest extends ClusterManagerNodeRequest { private final QueryGroup queryGroup; /** diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportCreateQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportCreateQueryGroupAction.java index 190ff17261bb4..dff9c429d63b0 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportCreateQueryGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportCreateQueryGroupAction.java @@ -9,43 +9,82 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService; -import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; + +import static org.opensearch.threadpool.ThreadPool.Names.SAME; + /** * Transport action to create QueryGroup * * @opensearch.experimental */ -public class TransportCreateQueryGroupAction extends HandledTransportAction { +public class TransportCreateQueryGroupAction extends TransportClusterManagerNodeAction { private final QueryGroupPersistenceService queryGroupPersistenceService; /** * Constructor for TransportCreateQueryGroupAction * - * @param actionName - action name + * @param threadPool - {@link ThreadPool} object * @param transportService - a {@link TransportService} object * @param actionFilters - a {@link ActionFilters} object + * @param indexNameExpressionResolver - {@link IndexNameExpressionResolver} object * @param queryGroupPersistenceService - a {@link QueryGroupPersistenceService} object */ @Inject public TransportCreateQueryGroupAction( - String actionName, + ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, QueryGroupPersistenceService queryGroupPersistenceService ) { - super(CreateQueryGroupAction.NAME, transportService, actionFilters, CreateQueryGroupRequest::new); + super( + CreateQueryGroupAction.NAME, + transportService, + queryGroupPersistenceService.getClusterService(), + threadPool, + actionFilters, + CreateQueryGroupRequest::new, + indexNameExpressionResolver + ); this.queryGroupPersistenceService = queryGroupPersistenceService; } @Override - protected void doExecute(Task task, CreateQueryGroupRequest request, ActionListener listener) { + protected void clusterManagerOperation( + CreateQueryGroupRequest request, + ClusterState clusterState, + ActionListener listener + ) { queryGroupPersistenceService.persistInClusterStateMetadata(request.getQueryGroup(), listener); } + + @Override + protected String executor() { + return SAME; + } + + @Override + protected CreateQueryGroupResponse read(StreamInput in) throws IOException { + return new CreateQueryGroupResponse(in); + } + + @Override + protected ClusterBlockException checkBlock(CreateQueryGroupRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java index a6aa2da8fdc08..09a0da7086b36 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportUpdateQueryGroupAction.java @@ -9,43 +9,81 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService; -import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; + +import static org.opensearch.threadpool.ThreadPool.Names.SAME; + /** * Transport action to update QueryGroup * * @opensearch.experimental */ -public class TransportUpdateQueryGroupAction extends HandledTransportAction { +public class TransportUpdateQueryGroupAction extends TransportClusterManagerNodeAction { private final QueryGroupPersistenceService queryGroupPersistenceService; /** * Constructor for TransportUpdateQueryGroupAction * - * @param actionName - action name + * @param threadPool - {@link ThreadPool} object * @param transportService - a {@link TransportService} object * @param actionFilters - a {@link ActionFilters} object + * @param indexNameExpressionResolver - {@link IndexNameExpressionResolver} object * @param queryGroupPersistenceService - a {@link QueryGroupPersistenceService} object */ @Inject public TransportUpdateQueryGroupAction( - String actionName, + ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, QueryGroupPersistenceService queryGroupPersistenceService ) { - super(UpdateQueryGroupAction.NAME, transportService, actionFilters, UpdateQueryGroupRequest::new); + super( + UpdateQueryGroupAction.NAME, + transportService, + queryGroupPersistenceService.getClusterService(), + threadPool, + actionFilters, + UpdateQueryGroupRequest::new, + indexNameExpressionResolver + ); this.queryGroupPersistenceService = queryGroupPersistenceService; } @Override - protected void doExecute(Task task, UpdateQueryGroupRequest request, ActionListener listener) { + protected void clusterManagerOperation( + UpdateQueryGroupRequest request, + ClusterState clusterState, + ActionListener listener + ) { queryGroupPersistenceService.updateInClusterStateMetadata(request, listener); } + + @Override + protected String executor() { + return SAME; + } + + @Override + protected UpdateQueryGroupResponse read(StreamInput in) throws IOException { + return new UpdateQueryGroupResponse(in); + } + + @Override + protected ClusterBlockException checkBlock(UpdateQueryGroupRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java index 048b599f095fd..18af58289be13 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/UpdateQueryGroupRequest.java @@ -8,8 +8,8 @@ package org.opensearch.plugin.wlm.action; -import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.cluster.metadata.QueryGroup; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -23,7 +23,7 @@ * * @opensearch.experimental */ -public class UpdateQueryGroupRequest extends ActionRequest { +public class UpdateQueryGroupRequest extends ClusterManagerNodeRequest { private final String name; private final MutableQueryGroupFragment mutableQueryGroupFragment; From ca40ba4646b3b5298c2c9e6e652df596048468e5 Mon Sep 17 00:00:00 2001 From: Rahul Karajgikar <50844303+rahulkarajgikar@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:51:21 +0530 Subject: [PATCH 275/357] Make multiple settings dynamic for tuning on larger clusters (#16347) Signed-off-by: Rahul Karajgikar --- CHANGELOG.md | 1 + .../cluster/coordination/Coordinator.java | 12 +- .../ElectionSchedulerFactory.java | 2 +- .../coordination/FollowersChecker.java | 12 +- .../cluster/coordination/LagDetector.java | 12 +- .../gateway/ShardsBatchGatewayAllocator.java | 11 +- .../CoordinationCheckerSettingsTests.java | 105 +++++++++++++++++- .../coordination/LagDetectorTests.java | 6 +- .../ShardsBatchGatewayAllocatorTests.java | 66 +++++++++++ 9 files changed, 212 insertions(+), 15 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/ShardsBatchGatewayAllocatorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c1485de3de2ee..0f95cb2484984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) - Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) - Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215)) +- Make multiple settings dynamic for tuning on larger clusters([#16347](https://github.com/opensearch-project/OpenSearch/pull/16347)) ### Deprecated diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 684a6b0c3eae5..6fee2037501e7 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -141,7 +141,8 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery "cluster.publish.timeout", TimeValue.timeValueMillis(30000), TimeValue.timeValueMillis(1), - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); private final Settings settings; @@ -164,7 +165,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final Random random; private final ElectionSchedulerFactory electionSchedulerFactory; private final SeedHostsResolver configuredHostsResolver; - private final TimeValue publishTimeout; + private TimeValue publishTimeout; private final TimeValue publishInfoTimeout; private final PublicationTransportHandler publicationHandler; private final LeaderChecker leaderChecker; @@ -247,6 +248,7 @@ public Coordinator( this.lastJoin = Optional.empty(); this.joinAccumulator = new InitialJoinAccumulator(); this.publishTimeout = PUBLISH_TIMEOUT_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(PUBLISH_TIMEOUT_SETTING, this::setPublishTimeout); this.publishInfoTimeout = PUBLISH_INFO_TIMEOUT_SETTING.get(settings); this.random = random; this.electionSchedulerFactory = new ElectionSchedulerFactory(settings, random, transportService.getThreadPool()); @@ -301,6 +303,7 @@ public Coordinator( ); this.lagDetector = new LagDetector( settings, + clusterSettings, transportService.getThreadPool(), n -> removeNode(n, "lagging"), transportService::getLocalNode @@ -319,6 +322,10 @@ public Coordinator( this.clusterSettings = clusterSettings; } + private void setPublishTimeout(TimeValue publishTimeout) { + this.publishTimeout = publishTimeout; + } + private ClusterFormationState getClusterFormationState() { return new ClusterFormationState( settings, @@ -1669,7 +1676,6 @@ public void onNodeAck(DiscoveryNode node, Exception e) { this.localNodeAckEvent = localNodeAckEvent; this.ackListener = ackListener; this.publishListener = publishListener; - this.timeoutHandler = singleNodeDiscovery ? null : transportService.getThreadPool().schedule(new Runnable() { @Override public void run() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/ElectionSchedulerFactory.java b/server/src/main/java/org/opensearch/cluster/coordination/ElectionSchedulerFactory.java index 828db5864d28b..1cc88c71c609b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/ElectionSchedulerFactory.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/ElectionSchedulerFactory.java @@ -214,7 +214,7 @@ protected void doRun() { if (isClosed.get()) { logger.debug("{} not starting election", this); } else { - logger.debug("{} starting election", this); + logger.debug("{} starting election with duration {}", this, duration); scheduleNextElection(duration, scheduledRunnable); scheduledRunnable.run(); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java index 2ec0dabd91786..ca414ef7c4fc8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java @@ -92,7 +92,8 @@ public class FollowersChecker { "cluster.fault_detection.follower_check.interval", TimeValue.timeValueMillis(1000), TimeValue.timeValueMillis(100), - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); // the timeout for each check sent to each node @@ -100,7 +101,7 @@ public class FollowersChecker { "cluster.fault_detection.follower_check.timeout", TimeValue.timeValueMillis(10000), TimeValue.timeValueMillis(1), - TimeValue.timeValueMillis(60000), + TimeValue.timeValueMillis(150000), Setting.Property.NodeScope, Setting.Property.Dynamic ); @@ -115,7 +116,7 @@ public class FollowersChecker { private final Settings settings; - private final TimeValue followerCheckInterval; + private TimeValue followerCheckInterval; private TimeValue followerCheckTimeout; private final int followerCheckRetryCount; private final BiConsumer onNodeFailure; @@ -148,6 +149,7 @@ public FollowersChecker( followerCheckInterval = FOLLOWER_CHECK_INTERVAL_SETTING.get(settings); followerCheckTimeout = FOLLOWER_CHECK_TIMEOUT_SETTING.get(settings); followerCheckRetryCount = FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(FOLLOWER_CHECK_INTERVAL_SETTING, this::setFollowerCheckInterval); clusterSettings.addSettingsUpdateConsumer(FOLLOWER_CHECK_TIMEOUT_SETTING, this::setFollowerCheckTimeout); updateFastResponseState(0, Mode.CANDIDATE); transportService.registerRequestHandler( @@ -167,6 +169,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti this.clusterManagerMetrics = clusterManagerMetrics; } + private void setFollowerCheckInterval(TimeValue followerCheckInterval) { + this.followerCheckInterval = followerCheckInterval; + } + private void setFollowerCheckTimeout(TimeValue followerCheckTimeout) { this.followerCheckTimeout = followerCheckTimeout; } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/LagDetector.java b/server/src/main/java/org/opensearch/cluster/coordination/LagDetector.java index eeb0800663d0a..969c121dc87cf 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/LagDetector.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/LagDetector.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -68,10 +69,11 @@ public class LagDetector { "cluster.follower_lag.timeout", TimeValue.timeValueMillis(90000), TimeValue.timeValueMillis(1), - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); - private final TimeValue clusterStateApplicationTimeout; + private TimeValue clusterStateApplicationTimeout; private final Consumer onLagDetected; private final Supplier localNodeSupplier; private final ThreadPool threadPool; @@ -79,12 +81,14 @@ public class LagDetector { public LagDetector( final Settings settings, + final ClusterSettings clusterSettings, final ThreadPool threadPool, final Consumer onLagDetected, final Supplier localNodeSupplier ) { this.threadPool = threadPool; this.clusterStateApplicationTimeout = CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING, this::setFollowerLagTimeout); this.onLagDetected = onLagDetected; this.localNodeSupplier = localNodeSupplier; } @@ -136,6 +140,10 @@ public String toString() { } } + private void setFollowerLagTimeout(TimeValue followerCheckLagTimeout) { + this.clusterStateApplicationTimeout = followerCheckLagTimeout; + } + @Override public String toString() { return "LagDetector{" diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java index d7c0a66ba3424..9c38ea1df8a41 100644 --- a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java @@ -72,7 +72,7 @@ public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator { public static final String ALLOCATOR_NAME = "shards_batch_gateway_allocator"; private static final Logger logger = LogManager.getLogger(ShardsBatchGatewayAllocator.class); - private final long maxBatchSize; + private long maxBatchSize; private static final short DEFAULT_SHARD_BATCH_SIZE = 2000; public static final String PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING_KEY = @@ -93,7 +93,8 @@ public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator { DEFAULT_SHARD_BATCH_SIZE, 1, 10000, - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.Dynamic ); /** @@ -172,6 +173,7 @@ public ShardsBatchGatewayAllocator( this.batchStartedAction = batchStartedAction; this.batchStoreAction = batchStoreAction; this.maxBatchSize = GATEWAY_ALLOCATOR_BATCH_SIZE.get(settings); + clusterSettings.addSettingsUpdateConsumer(GATEWAY_ALLOCATOR_BATCH_SIZE, this::setMaxBatchSize); this.primaryShardsBatchGatewayAllocatorTimeout = PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(PRIMARY_BATCH_ALLOCATOR_TIMEOUT_SETTING, this::setPrimaryBatchAllocatorTimeout); this.replicaShardsBatchGatewayAllocatorTimeout = REPLICA_BATCH_ALLOCATOR_TIMEOUT_SETTING.get(settings); @@ -402,6 +404,7 @@ else if (shardRouting.primary() == primary) { Iterator iterator = newShardsToBatch.values().iterator(); assert maxBatchSize > 0 : "Shards batch size must be greater than 0"; + logger.debug("Using async fetch batch size {}", maxBatchSize); long batchSize = maxBatchSize; Map perBatchShards = new HashMap<>(); while (iterator.hasNext()) { @@ -906,6 +909,10 @@ public int getNumberOfStoreShardBatches() { return batchIdToStoreShardBatch.size(); } + private void setMaxBatchSize(long maxBatchSize) { + this.maxBatchSize = maxBatchSize; + } + protected void setPrimaryBatchAllocatorTimeout(TimeValue primaryShardsBatchGatewayAllocatorTimeout) { this.primaryShardsBatchGatewayAllocatorTimeout = primaryShardsBatchGatewayAllocatorTimeout; } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java index 56bd2d94dce84..8e8e71ad33e75 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationCheckerSettingsTests.java @@ -14,7 +14,10 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchSingleNodeTestCase; +import static org.opensearch.cluster.coordination.Coordinator.PUBLISH_TIMEOUT_SETTING; +import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING; import static org.opensearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING; +import static org.opensearch.cluster.coordination.LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING; import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING; import static org.opensearch.common.unit.TimeValue.timeValueSeconds; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -42,10 +45,10 @@ public void testFollowerCheckTimeoutValueUpdate() { public void testFollowerCheckTimeoutMaxValue() { Setting setting1 = FOLLOWER_CHECK_TIMEOUT_SETTING; - Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "61s").build(); + Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "151s").build(); assertThrows( - "failed to parse value [61s] for setting [" + setting1.getKey() + "], must be <= [60000ms]", + "failed to parse value [151s] for setting [" + setting1.getKey() + "], must be <= [150000ms]", IllegalArgumentException.class, () -> { client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet(); @@ -66,6 +69,38 @@ public void testFollowerCheckTimeoutMinValue() { ); } + public void testFollowerCheckIntervalValueUpdate() { + Setting setting1 = FOLLOWER_CHECK_INTERVAL_SETTING; + Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "10s").build(); + try { + ClusterUpdateSettingsResponse response = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(timeSettings1) + .execute() + .actionGet(); + assertAcked(response); + assertEquals(timeValueSeconds(10), setting1.get(response.getPersistentSettings())); + } finally { + // cleanup + timeSettings1 = Settings.builder().putNull(setting1.getKey()).build(); + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet(); + } + } + + public void testFollowerCheckIntervalMinValue() { + Setting setting1 = FOLLOWER_CHECK_INTERVAL_SETTING; + Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "10ms").build(); + + assertThrows( + "failed to parse value [10ms] for setting [" + setting1.getKey() + "], must be >= [100ms]", + IllegalArgumentException.class, + () -> { + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet(); + } + ); + } + public void testLeaderCheckTimeoutValueUpdate() { Setting setting1 = LEADER_CHECK_TIMEOUT_SETTING; Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "60s").build(); @@ -110,4 +145,70 @@ public void testLeaderCheckTimeoutMinValue() { } ); } + + public void testClusterPublishTimeoutValueUpdate() { + Setting setting1 = PUBLISH_TIMEOUT_SETTING; + Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "60s").build(); + try { + ClusterUpdateSettingsResponse response = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(timeSettings1) + .execute() + .actionGet(); + assertAcked(response); + assertEquals(timeValueSeconds(60), setting1.get(response.getPersistentSettings())); + } finally { + // cleanup + timeSettings1 = Settings.builder().putNull(setting1.getKey()).build(); + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet(); + } + } + + public void testClusterPublishTimeoutMinValue() { + Setting setting1 = PUBLISH_TIMEOUT_SETTING; + Settings timeSettings1 = Settings.builder().put(setting1.getKey(), "0s").build(); + + assertThrows( + "failed to parse value [0s] for setting [" + setting1.getKey() + "], must be >= [1ms]", + IllegalArgumentException.class, + () -> { + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(timeSettings1).execute().actionGet(); + } + ); + } + + public void testLagDetectorTimeoutUpdate() { + Setting setting1 = CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING; + Settings lagDetectorTimeout = Settings.builder().put(setting1.getKey(), "30s").build(); + try { + ClusterUpdateSettingsResponse response = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(lagDetectorTimeout) + .execute() + .actionGet(); + + assertAcked(response); + assertEquals(timeValueSeconds(30), setting1.get(response.getPersistentSettings())); + } finally { + // cleanup + lagDetectorTimeout = Settings.builder().putNull(setting1.getKey()).build(); + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(lagDetectorTimeout).execute().actionGet(); + } + } + + public void testLagDetectorTimeoutMinValue() { + Setting setting1 = CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING; + Settings lagDetectorTimeout = Settings.builder().put(setting1.getKey(), "0s").build(); + + assertThrows( + "failed to parse value [0s] for setting [" + setting1.getKey() + "], must be >= [1ms]", + IllegalArgumentException.class, + () -> { + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(lagDetectorTimeout).execute().actionGet(); + } + ); + } + } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/LagDetectorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/LagDetectorTests.java index adffa27e9bc1a..315e5d6224227 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/LagDetectorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/LagDetectorTests.java @@ -32,6 +32,7 @@ package org.opensearch.cluster.coordination; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; @@ -70,8 +71,9 @@ public void setupFixture() { } else { followerLagTimeout = CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING.get(Settings.EMPTY); } - - lagDetector = new LagDetector(settingsBuilder.build(), deterministicTaskQueue.getThreadPool(), failedNodes::add, () -> localNode); + Settings settings = settingsBuilder.build(); + final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + lagDetector = new LagDetector(settings, clusterSettings, deterministicTaskQueue.getThreadPool(), failedNodes::add, () -> localNode); localNode = CoordinationStateTests.createNode("local"); node1 = CoordinationStateTests.createNode("node1"); diff --git a/server/src/test/java/org/opensearch/gateway/ShardsBatchGatewayAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/ShardsBatchGatewayAllocatorTests.java new file mode 100644 index 0000000000000..59fb6e2b940ba --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/ShardsBatchGatewayAllocatorTests.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import static org.opensearch.gateway.ShardsBatchGatewayAllocator.GATEWAY_ALLOCATOR_BATCH_SIZE; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class ShardsBatchGatewayAllocatorTests extends OpenSearchSingleNodeTestCase { + public void testBatchSizeValueUpdate() { + Setting setting1 = GATEWAY_ALLOCATOR_BATCH_SIZE; + Settings batchSizeSetting = Settings.builder().put(setting1.getKey(), "3000").build(); + try { + ClusterUpdateSettingsResponse response = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(batchSizeSetting) + .execute() + .actionGet(); + + assertAcked(response); + assertThat(setting1.get(response.getPersistentSettings()), equalTo(3000L)); + } finally { + // cleanup + batchSizeSetting = Settings.builder().putNull(setting1.getKey()).build(); + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(batchSizeSetting).execute().actionGet(); + } + } + + public void testBatchSizeMaxValue() { + Setting setting1 = GATEWAY_ALLOCATOR_BATCH_SIZE; + Settings batchSizeSetting = Settings.builder().put(setting1.getKey(), "11000").build(); + + assertThrows( + "failed to parse value [11000] for setting [" + setting1.getKey() + "], must be <= [10000]", + IllegalArgumentException.class, + () -> { + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(batchSizeSetting).execute().actionGet(); + } + ); + } + + public void testBatchSizeMinValue() { + Setting setting1 = GATEWAY_ALLOCATOR_BATCH_SIZE; + Settings batchSizeSetting = Settings.builder().put(setting1.getKey(), "0").build(); + + assertThrows( + "failed to parse value [0] for setting [" + setting1.getKey() + "], must be >= [1]", + IllegalArgumentException.class, + () -> { + client().admin().cluster().prepareUpdateSettings().setPersistentSettings(batchSizeSetting).execute().actionGet(); + } + ); + } +} From 9489a21a7466dbc6320b4b53c828ff3bfbce2b22 Mon Sep 17 00:00:00 2001 From: "Spencer G. Jones" Date: Tue, 22 Oct 2024 22:44:24 -0700 Subject: [PATCH 276/357] =?UTF-8?q?Add=20new=20parameters=20to=20snapshot?= =?UTF-8?q?=20restore=20to=20rename=20the=20restored=20aliases=E2=80=A6=20?= =?UTF-8?q?(#16292)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add new parameters to snapshot restore to rename the restored aliases similar to the existing parameters to rename indexes Signed-off-by: Spencer G. Jones * Fix comment. Update changelog. Signed-off-by: Spencer G. Jones * New parameters needs to only used for new version Signed-off-by: Spencer G. Jones * Add missing equals and hash implemenation for new parameters Signed-off-by: Spencer G. Jones * Add some tests Signed-off-by: Spencer G. Jones * Add some more tests Signed-off-by: Spencer G. Jones * Use CountDownLatch Signed-off-by: Spencer G. Jones * Add two more tests. Refactoring and cleanup. Signed-off-by: Spencer G. Jones * Use CURRENT version to pass backward compatibility tests. Change to V2.18 later once it is backported into that version. Signed-off-by: Spencer G. Jones * Refactoring Signed-off-by: Spencer G. Jones * Overwriting aliases variable causes test failures for reasons I do not understand. Also some refactoring. Signed-off-by: Spencer G. Jones * Convert to paramaterized tests Signed-off-by: Spencer G. Jones --------- Signed-off-by: Spencer G. Jones Signed-off-by: Daniel Widdis Co-authored-by: Daniel Widdis --- CHANGELOG.md | 2 +- .../restore/RestoreSnapshotRequest.java | 83 +++++ .../RestoreSnapshotRequestBuilder.java | 28 ++ .../opensearch/snapshots/RestoreService.java | 30 +- .../restore/RestoreSnapshotRequestTests.java | 6 + .../snapshots/RestoreServiceIntegTests.java | 297 ++++++++++++++++++ .../snapshots/SnapshotRequestsTests.java | 4 + .../snapshots/SnapshotResiliencyTests.java | 70 ++++- 8 files changed, 506 insertions(+), 14 deletions(-) create mode 100644 server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f95cb2484984..332dad2a7370a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) +- Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292)) - Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)) - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) -- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index f3110cc8f20a5..42c64e04268e3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -112,6 +112,8 @@ private static StorageType fromString(String string) { private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); private String renamePattern; private String renameReplacement; + private String renameAliasPattern; + private String renameAliasReplacement; private boolean waitForCompletion; private boolean includeGlobalState = false; private boolean partial = false; @@ -164,6 +166,13 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_17_0)) { sourceRemoteTranslogRepository = in.readOptionalString(); } + // TODO: change to V_2_18_0 once this is backported into that version + if (in.getVersion().onOrAfter(Version.CURRENT)) { + renameAliasPattern = in.readOptionalString(); + } + if (in.getVersion().onOrAfter(Version.CURRENT)) { + renameAliasReplacement = in.readOptionalString(); + } } @Override @@ -191,6 +200,13 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalString(sourceRemoteTranslogRepository); } + // TODO: change to V_2_18_0 once this is backported into that version + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalString(renameAliasPattern); + } + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalString(renameAliasReplacement); + } } @Override @@ -361,6 +377,51 @@ public String renameReplacement() { return renameReplacement; } + /** + * Sets rename pattern that should be applied to restored indices' alias. + *

    + * Alias that match the rename pattern will be renamed according to {@link #renameAliasReplacement(String)}. The + * rename pattern is applied according to the {@link java.util.regex.Matcher#appendReplacement(StringBuffer, String)} + * If two or more aliases are renamed into the same name, they will be merged. + * + * @param renameAliasPattern rename pattern + * @return this request + */ + public RestoreSnapshotRequest renameAliasPattern(String renameAliasPattern) { + this.renameAliasPattern = renameAliasPattern; + return this; + } + + /** + * Returns rename alias pattern + * + * @return rename alias pattern + */ + public String renameAliasPattern() { + return renameAliasPattern; + } + + /** + * Sets rename alias replacement + *

    + * See {@link #renameAliasPattern(String)} for more information. + * + * @param renameAliasReplacement rename replacement + */ + public RestoreSnapshotRequest renameAliasReplacement(String renameAliasReplacement) { + this.renameAliasReplacement = renameAliasReplacement; + return this; + } + + /** + * Returns rename alias replacement + * + * @return rename alias replacement + */ + public String renameAliasReplacement() { + return renameAliasReplacement; + } + /** * If this parameter is set to true the operation will wait for completion of restore process before returning. * @@ -625,6 +686,18 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed rename_replacement"); } + } else if (name.equals("rename_alias_pattern")) { + if (entry.getValue() instanceof String) { + renameAliasPattern((String) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed rename_alias_pattern"); + } + } else if (name.equals("rename_alias_replacement")) { + if (entry.getValue() instanceof String) { + renameAliasReplacement((String) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed rename_alias_replacement"); + } } else if (name.equals("index_settings")) { if (!(entry.getValue() instanceof Map)) { throw new IllegalArgumentException("malformed index_settings section"); @@ -685,6 +758,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } + if (renameAliasPattern != null) { + builder.field("rename_alias_pattern", renameAliasPattern); + } + if (renameAliasReplacement != null) { + builder.field("rename_alias_replacement", renameAliasReplacement); + } builder.field("include_global_state", includeGlobalState); builder.field("partial", partial); builder.field("include_aliases", includeAliases); @@ -733,6 +812,8 @@ public boolean equals(Object o) { && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(renamePattern, that.renamePattern) && Objects.equals(renameReplacement, that.renameReplacement) + && Objects.equals(renameAliasPattern, that.renameAliasPattern) + && Objects.equals(renameAliasReplacement, that.renameAliasReplacement) && Objects.equals(indexSettings, that.indexSettings) && Arrays.equals(ignoreIndexSettings, that.ignoreIndexSettings) && Objects.equals(snapshotUuid, that.snapshotUuid) @@ -751,6 +832,8 @@ public int hashCode() { indicesOptions, renamePattern, renameReplacement, + renameAliasPattern, + renameAliasReplacement, waitForCompletion, includeGlobalState, partial, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index 53c9557a621b7..038d62ad7f4cb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -144,6 +144,34 @@ public RestoreSnapshotRequestBuilder setRenameReplacement(String renameReplaceme return this; } + /** + * Sets rename pattern that should be applied to restored indices' aliases. + *

    + * Aliases that match the rename pattern will be renamed according to {@link #setRenameAliasReplacement(String)}. The + * rename pattern is applied according to the {@link java.util.regex.Matcher#appendReplacement(StringBuffer, String)} + * The request will fail if two or more alias will be renamed into the same name. + * + * @param renameAliasPattern rename alias pattern + * @return this builder + */ + public RestoreSnapshotRequestBuilder setRenameAliasPattern(String renameAliasPattern) { + request.renameAliasPattern(renameAliasPattern); + return this; + } + + /** + * Sets rename replacement + *

    + * See {@link #setRenameAliasPattern(String)} for more information. + * + * @param renameAliasReplacement rename alias replacement + * @return this builder + */ + public RestoreSnapshotRequestBuilder setRenameAliasReplacement(String renameAliasReplacement) { + request.renameAliasReplacement(renameAliasReplacement); + return this; + } + /** * If this parameter is set to true the operation will wait for completion of restore process before returning. * diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 79a70d835f773..88eff93e51b38 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -111,6 +111,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.util.Collections.unmodifiableSet; @@ -486,9 +487,7 @@ public ClusterState execute(ClusterState currentState) { // Remove all aliases - they shouldn't be restored indexMdBuilder.removeAllAliases(); } else { - for (final String alias : snapshotIndexMetadata.getAliases().keySet()) { - aliases.add(alias); - } + applyAliasesWithRename(snapshotIndexMetadata, indexMdBuilder, aliases); } IndexMetadata updatedIndexMetadata = indexMdBuilder.build(); if (partial) { @@ -533,9 +532,7 @@ public ClusterState execute(ClusterState currentState) { indexMdBuilder.putAlias(alias); } } else { - for (final String alias : snapshotIndexMetadata.getAliases().keySet()) { - aliases.add(alias); - } + applyAliasesWithRename(snapshotIndexMetadata, indexMdBuilder, aliases); } final Settings.Builder indexSettingsBuilder = Settings.builder() .put(snapshotIndexMetadata.getSettings()) @@ -665,6 +662,27 @@ private void checkAliasNameConflicts(Map renamedIndices, Set aliases + ) { + if (request.renameAliasPattern() == null || request.renameAliasReplacement() == null) { + aliases.addAll(snapshotIndexMetadata.getAliases().keySet()); + } else { + Pattern renameAliasPattern = Pattern.compile(request.renameAliasPattern()); + for (final Map.Entry alias : snapshotIndexMetadata.getAliases().entrySet()) { + String currentAliasName = alias.getKey(); + indexMdBuilder.removeAlias(currentAliasName); + String newAliasName = renameAliasPattern.matcher(currentAliasName) + .replaceAll(request.renameAliasReplacement()); + AliasMetadata newAlias = AliasMetadata.newAliasMetadata(alias.getValue(), newAliasName); + indexMdBuilder.putAlias(newAlias); + aliases.add(newAliasName); + } + } + } + private String[] getIgnoreSettingsInternal() { // for non-remote store enabled domain, we will remove all the remote store // related index settings present in the snapshot. diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index c3de3413edd13..04cc45f3477c6 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -71,6 +71,12 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { if (randomBoolean()) { instance.renameReplacement(randomUnicodeOfLengthBetween(1, 100)); } + if (randomBoolean()) { + instance.renameAliasPattern(randomUnicodeOfLengthBetween(1, 100)); + } + if (randomBoolean()) { + instance.renameAliasReplacement(randomUnicodeOfLengthBetween(1, 100)); + } instance.partial(randomBoolean()); instance.includeAliases(randomBoolean()); diff --git a/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java new file mode 100644 index 0000000000000..92da980d70f34 --- /dev/null +++ b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java @@ -0,0 +1,297 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.snapshots; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.action.admin.indices.close.CloseIndexRequest; +import org.opensearch.action.admin.indices.close.CloseIndexResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.opensearch.action.admin.indices.open.OpenIndexRequest; +import org.opensearch.action.admin.indices.open.OpenIndexResponse; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchSingleNodeTestCase; +import org.junit.After; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class RestoreServiceIntegTests extends OpenSearchSingleNodeTestCase { + private final String indexName = "index_1"; + private final String renamedIndexName = "index_2"; + private final String aliasName = "alias_1"; + private final String renamedAliasName = "alias_2"; + private final String repoName = "repo_1"; + private final String snapShotName = "snap_1"; + private final int waitInSeconds = 60; + private boolean exists; + private boolean closed; + private boolean includeAlias; + private boolean renameAliases; + private boolean renameIndexes; + + public RestoreServiceIntegTests(TestCase testCase) { + this.exists = testCase.exists; + this.closed = testCase.closed; + this.includeAlias = testCase.includeAlias; + this.renameAliases = testCase.renameAliases; + this.renameIndexes = testCase.renameIndexes; + } + + public static class TestCase { + public boolean exists; + public boolean closed; + public boolean includeAlias; + public boolean renameAliases; + public boolean renameIndexes; + + public TestCase(boolean exists, boolean closed, boolean includeAlias, boolean renameAliases, boolean renameIndexes) { + this.exists = exists; + this.closed = closed; + this.includeAlias = includeAlias; + this.renameAliases = renameAliases; + this.renameIndexes = renameIndexes; + } + + public String toString() { + return String.join( + " and ", + new String[] { + exists ? "target index exists and is" + (closed ? "closed" : "open") : "doesn't exist", + includeAlias ? "including aliases" : "not including aliases", + renameIndexes ? "renaming indexes" : "not renaming indexes", + renameAliases ? "renaming aliases" : "not renaming aliases" } + ); + } + } + + @ParametersFactory + public static Collection parameters() { + return Arrays.asList( + new Object[] { new TestCase(false, false, true, true, true) }, + new Object[] { new TestCase(false, false, false, true, true) }, + new Object[] { new TestCase(false, false, true, false, false) }, + new Object[] { new TestCase(false, false, false, false, false) }, + new Object[] { new TestCase(true, false, true, true, true) }, + new Object[] { new TestCase(true, false, false, true, true) }, + new Object[] { new TestCase(true, true, true, true, true) }, + new Object[] { new TestCase(true, true, false, true, true) }, + new Object[] { new TestCase(true, false, false, false, false) }, + new Object[] { new TestCase(true, false, true, false, false) }, + new Object[] { new TestCase(true, true, false, false, false) }, + new Object[] { new TestCase(true, true, true, false, false) } + ); + } + + @After + public void cleanup() throws InterruptedException { + final CountDownLatch allDeleted = new CountDownLatch(3); + for (String indexName : new String[] { indexName, renamedIndexName }) { + final StepListener existsIndexResponseStepListener = new StepListener<>(); + client().admin().indices().exists(new IndicesExistsRequest(indexName), existsIndexResponseStepListener); + continueOrDie(existsIndexResponseStepListener, resp -> { + if (resp.isExists()) { + final StepListener deleteIndexResponseStepListener = new StepListener<>(); + client().admin().indices().delete(new DeleteIndexRequest(indexName), deleteIndexResponseStepListener); + continueOrDie(deleteIndexResponseStepListener, ignored -> allDeleted.countDown()); + } else { + allDeleted.countDown(); + } + }); + } + + final StepListener snapStatusResponseStepListener = new StepListener<>(); + client().admin().cluster().getSnapshots(new GetSnapshotsRequest(repoName), snapStatusResponseStepListener); + continueOrDie(snapStatusResponseStepListener, resp -> { + if (resp.getSnapshots().stream().anyMatch(s -> s.snapshotId().getName().equals(snapShotName))) { + final StepListener deleteSnapResponseStepListener = new StepListener<>(); + client().admin() + .cluster() + .deleteSnapshot(new DeleteSnapshotRequest(repoName, snapShotName), deleteSnapResponseStepListener); + continueOrDie(deleteSnapResponseStepListener, ignored -> allDeleted.countDown()); + } else { + allDeleted.countDown(); + } + }); + + allDeleted.await(waitInSeconds, TimeUnit.SECONDS); + } + + public void testRestoreWithRename() throws Exception { + + assert this.exists || !this.closed; // index close state doesn't exist when the index doesn't exist - so only permit one value of + // closed to avoid pointless duplicate tests + final boolean expectSuccess = !this.exists || this.closed; + final int documents = randomIntBetween(1, 100); + + this.createIndex(indexName); + if (this.exists && this.renameIndexes) { + this.createIndex(renamedIndexName); + } + + final StepListener putRepositoryResponseStepListener = new StepListener<>(); + Settings.Builder settings = Settings.builder().put("location", randomAlphaOfLength(10)); + OpenSearchIntegTestCase.putRepository( + client().admin().cluster(), + repoName, + FsRepository.TYPE, + settings, + putRepositoryResponseStepListener + ); + + final StepListener createAliasResponseStepListener = new StepListener<>(); + client().admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.add().alias(aliasName).index(indexName)), + createAliasResponseStepListener + ); + + final CountDownLatch isDocumentFinished = new CountDownLatch(1); + continueOrDie(createAliasResponseStepListener, ignored -> { + final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + bulkRequest.add(new IndexRequest(indexName).source(Collections.singletonMap("foo", "bar" + i))); + } + final StepListener bulkResponseStepListener = new StepListener<>(); + client().bulk(bulkRequest, bulkResponseStepListener); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + isDocumentFinished.countDown(); + }); + }); + + isDocumentFinished.await(waitInSeconds, TimeUnit.SECONDS); + + if (this.closed) { + final CountDownLatch isClosed = new CountDownLatch(1); + final StepListener closeIndexResponseStepListener = new StepListener<>(); + final String indexToClose = this.renameIndexes ? renamedIndexName : indexName; + client().admin().indices().close(new CloseIndexRequest(indexToClose), closeIndexResponseStepListener); + + continueOrDie(closeIndexResponseStepListener, ignored -> { isClosed.countDown(); }); + isClosed.await(waitInSeconds, TimeUnit.SECONDS); + } + + final StepListener createSnapshotResponseStepListener = new StepListener<>(); + continueOrDie(putRepositoryResponseStepListener, ignored -> { + client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapShotName) + .setWaitForCompletion(true) + .setPartial(true) + .execute(createSnapshotResponseStepListener); + }); + + final CountDownLatch isRestorable = new CountDownLatch(1); + + if (!this.exists && !this.renameIndexes) { + final StepListener deleteIndexResponseStepListener = new StepListener<>(); + continueOrDie(createSnapshotResponseStepListener, ignored -> { + client().admin().indices().delete(new DeleteIndexRequest(indexName), deleteIndexResponseStepListener); + }); + continueOrDie(deleteIndexResponseStepListener, ignored -> isRestorable.countDown()); + } else { + continueOrDie(createSnapshotResponseStepListener, ignored -> isRestorable.countDown()); + } + + isRestorable.await(waitInSeconds, TimeUnit.SECONDS); + + final StepListener restoreSnapshotResponseStepListener = new StepListener<>(); + final CountDownLatch isRestored = new CountDownLatch(1); + RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(repoName, snapShotName).includeAliases(this.includeAlias) + .waitForCompletion(true); + if (this.renameAliases) { + restoreSnapshotRequest = restoreSnapshotRequest.renameAliasPattern("1").renameAliasReplacement("2"); + } + if (this.renameIndexes) { + restoreSnapshotRequest = restoreSnapshotRequest.renamePattern("1").renameReplacement("2"); + } + client().admin().cluster().restoreSnapshot(restoreSnapshotRequest, restoreSnapshotResponseStepListener); + + restoreSnapshotResponseStepListener.whenComplete(ignored -> { + isRestored.countDown(); + assertTrue("unexpected sucesssful restore", expectSuccess); + }, e -> { + isRestored.countDown(); + if (expectSuccess) { + throw new RuntimeException(e); + } + }); + + isRestored.await(waitInSeconds, TimeUnit.SECONDS); + + if (expectSuccess) { + final String indexToSearch = this.renameIndexes ? renamedIndexName : indexName; + final String aliasToSearch = this.renameAliases ? renamedAliasName : aliasName; + + if (this.closed) { + final CountDownLatch isOpened = new CountDownLatch(1); + final StepListener openIndexResponseStepListener = new StepListener<>(); + client().admin().indices().open(new OpenIndexRequest(indexToSearch).waitForActiveShards(1), openIndexResponseStepListener); + continueOrDie(openIndexResponseStepListener, ignored -> { isOpened.countDown(); }); + + isOpened.await(waitInSeconds, TimeUnit.SECONDS); + } + + final CountDownLatch isSearchDone = new CountDownLatch(this.includeAlias ? 2 : 1); + final StepListener searchIndexResponseListener = new StepListener<>(); + final StepListener searchAliasResponseListener = new StepListener<>(); + client().search( + new SearchRequest(indexToSearch).source(new SearchSourceBuilder().size(0).trackTotalHits(true)), + searchIndexResponseListener + ); + continueOrDie(searchIndexResponseListener, ignored -> { isSearchDone.countDown(); }); + if (this.includeAlias) { + client().search( + new SearchRequest(aliasToSearch).source(new SearchSourceBuilder().size(0).trackTotalHits(true)), + searchAliasResponseListener + ); + continueOrDie(searchAliasResponseListener, ignored -> { isSearchDone.countDown(); }); + } + + isSearchDone.await(waitInSeconds, TimeUnit.SECONDS); + + assertEquals(documents, Objects.requireNonNull(searchIndexResponseListener.result().getHits().getTotalHits()).value); + if (this.includeAlias) { + assertEquals(documents, Objects.requireNonNull(searchAliasResponseListener.result().getHits().getTotalHits()).value); + } + } + } + + private static void continueOrDie(StepListener listener, CheckedConsumer onResponse) { + listener.whenComplete(onResponse, e -> { throw new AssertionError(e); }); + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java index a00c74f669eac..87ab95fef6a53 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotRequestsTests.java @@ -77,6 +77,8 @@ public void testRestoreSnapshotRequestParsing() throws IOException { builder.field("allow_no_indices", indicesOptions.allowNoIndices()); builder.field("rename_pattern", "rename-from"); builder.field("rename_replacement", "rename-to"); + builder.field("rename_alias_pattern", "alias-rename-from"); + builder.field("rename_alias_replacement", "alias-rename-to"); boolean partial = randomBoolean(); builder.field("partial", partial); builder.startObject("settings").field("set1", "val1").endObject(); @@ -103,6 +105,8 @@ public void testRestoreSnapshotRequestParsing() throws IOException { assertArrayEquals(request.indices(), new String[] { "foo", "bar", "baz" }); assertEquals("rename-from", request.renamePattern()); assertEquals("rename-to", request.renameReplacement()); + assertEquals("alias-rename-from", request.renameAliasPattern()); + assertEquals("alias-rename-to", request.renameAliasReplacement()); assertEquals(partial, request.partial()); assertArrayEquals(request.ignoreIndexSettings(), new String[] { "set2", "set3" }); boolean expectedIgnoreAvailable = includeIgnoreUnavailable diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index d17e661615b0d..d21282ff0441f 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -66,6 +66,9 @@ import org.opensearch.action.admin.cluster.state.ClusterStateRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.cluster.state.TransportClusterStateAction; +import org.opensearch.action.admin.indices.alias.IndicesAliasesAction; +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.action.admin.indices.alias.TransportIndicesAliasesAction; import org.opensearch.action.admin.indices.create.CreateIndexAction; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -141,6 +144,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataCreateIndexService; import org.opensearch.cluster.metadata.MetadataDeleteIndexService; +import org.opensearch.cluster.metadata.MetadataIndexAliasesService; import org.opensearch.cluster.metadata.MetadataIndexUpgradeService; import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.node.DiscoveryNode; @@ -958,6 +962,7 @@ public void testConcurrentSnapshotRestoreAndDeleteOther() { String repoName = "repo"; String snapshotName = "snapshot"; final String index = "test"; + final String alias = "test_alias"; final int shards = randomIntBetween(1, 10); TestClusterNodes.TestClusterNode clusterManagerNode = testClusterNodes.currentClusterManager( @@ -967,9 +972,8 @@ public void testConcurrentSnapshotRestoreAndDeleteOther() { final StepListener createSnapshotResponseStepListener = new StepListener<>(); final int documentsFirstSnapshot = randomIntBetween(0, 100); - continueOrDie( - createRepoAndIndex(repoName, index, shards), + createRepoAndIndexAndAlias(repoName, index, shards, alias), createIndexResponse -> indexNDocuments( documentsFirstSnapshot, index, @@ -1009,19 +1013,27 @@ public void testConcurrentSnapshotRestoreAndDeleteOther() { .cluster() .restoreSnapshot( new RestoreSnapshotRequest(repoName, secondSnapshotName).waitForCompletion(true) + .includeAliases(true) .renamePattern("(.+)") - .renameReplacement("restored_$1"), + .renameReplacement("restored_$1") + .renameAliasPattern("(.+)") + .renameAliasReplacement("restored_alias_$1"), restoreSnapshotResponseListener ) ); }); - final StepListener searchResponseListener = new StepListener<>(); + final StepListener searchIndexResponseListener = new StepListener<>(); + final StepListener searchAliasResponseListener = new StepListener<>(); continueOrDie(restoreSnapshotResponseListener, restoreSnapshotResponse -> { assertEquals(shards, restoreSnapshotResponse.getRestoreInfo().totalShards()); client().search( new SearchRequest("restored_" + index).source(new SearchSourceBuilder().size(0).trackTotalHits(true)), - searchResponseListener + searchIndexResponseListener + ); + client().search( + new SearchRequest("restored_alias_" + alias).source(new SearchSourceBuilder().size(0).trackTotalHits(true)), + searchAliasResponseListener ); }); @@ -1029,7 +1041,11 @@ public void testConcurrentSnapshotRestoreAndDeleteOther() { assertEquals( documentsFirstSnapshot + documentsSecondSnapshot, - Objects.requireNonNull(searchResponseListener.result().getHits().getTotalHits()).value + Objects.requireNonNull(searchIndexResponseListener.result().getHits().getTotalHits()).value + ); + assertEquals( + documentsFirstSnapshot + documentsSecondSnapshot, + Objects.requireNonNull(searchAliasResponseListener.result().getHits().getTotalHits()).value ); assertThat(deleteSnapshotStepListener.result().isAcknowledged(), is(true)); assertThat(restoreSnapshotResponseListener.result().getRestoreInfo().failedShards(), is(0)); @@ -1520,6 +1536,22 @@ private StepListener createRepoAndIndex(String repoName, St return createIndexResponseStepListener; } + private StepListener createRepoAndIndexAndAlias(String repoName, String index, int shards, String alias) { + final StepListener createAliasListener = new StepListener<>(); + + continueOrDie( + createRepoAndIndex(repoName, index, shards), + acknowledgedResponse -> client().admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.add().index(index).alias(alias)), + createAliasListener + ) + ); + + return createAliasListener; + } + private void clearDisruptionsAndAwaitSync() { testClusterNodes.clearNetworkDisruptions(); stabilize(); @@ -2171,6 +2203,30 @@ public void onFailure(final Exception e) { indexNameExpressionResolver ) ); + final MetadataDeleteIndexService metadataDeleteIndexService = new MetadataDeleteIndexService( + settings, + clusterService, + allocationService + ); + final MetadataIndexAliasesService metadataIndexAliasesService = new MetadataIndexAliasesService( + clusterService, + indicesService, + new AliasValidator(), + metadataDeleteIndexService, + namedXContentRegistry + ); + actions.put( + IndicesAliasesAction.INSTANCE, + new TransportIndicesAliasesAction( + transportService, + clusterService, + threadPool, + metadataIndexAliasesService, + actionFilters, + indexNameExpressionResolver, + new RequestValidators<>(Collections.emptyList()) + ) + ); final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings, clusterService); mappingUpdatedAction.setClient(client); final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction( @@ -2337,7 +2393,7 @@ public void onFailure(final Exception e) { transportService, clusterService, threadPool, - new MetadataDeleteIndexService(settings, clusterService, allocationService), + metadataDeleteIndexService, actionFilters, indexNameExpressionResolver, new DestructiveOperations(settings, clusterSettings) From bb1359f224158e94523ad1a2ecf5429244a8144f Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:09:10 +0530 Subject: [PATCH 277/357] Disallow snapshot deletion while a v2 snapshot is in progress (#16430) --------- Signed-off-by: Gaurav Bafna --- .../snapshots/ConcurrentSnapshotsV2IT.java | 5 ++--- .../snapshots/SnapshotsService.java | 21 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index 78497cac41d46..ab5b22c69b517 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -529,14 +529,13 @@ public void testDeleteWhileV2CreateOngoing() throws Exception { awaitNumberOfSnapshotsInProgress(1); ActionFuture a = startDeleteSnapshot(repoName, "snapshot-v1"); + expectThrows(ConcurrentSnapshotExecutionException.class, a::actionGet); unblockNode(repoName, clusterManagerName); CreateSnapshotResponse csr = snapshotFuture.actionGet(); assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0); - assertTrue(a.actionGet().isAcknowledged()); List snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(); - assertEquals(1, snapInfo.size()); - assertThat(snapInfo, contains(csr.getSnapshotInfo())); + assertEquals(2, snapInfo.size()); } @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205") diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index ff1395c600ac0..0972f5dad0fa2 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -619,10 +619,8 @@ public void onResponse(RepositoryData repositoryData) { } cleanOrphanTimestamp(repositoryName, repositoryData); logger.info("created snapshot-v2 [{}] in repository [{}]", repositoryName, snapshotName); + leaveRepoLoop(repositoryName); listener.onResponse(snapshotInfo); - // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations - // can get queued . This is triggering them. - runNextQueuedOperation(repositoryData, repositoryName, true); } @Override @@ -1021,10 +1019,8 @@ public void onResponse(RepositoryData repositoryData) { return; } logger.info("snapshot-v2 clone [{}] completed successfully", snapshot); + leaveRepoLoop(repositoryName); listener.onResponse(null); - // For snapshot-v2, we don't allow concurrent snapshots . But meanwhile non-v2 snapshot operations - // can get queued . This is triggering them. - runNextQueuedOperation(repositoryData, repositoryName, true); } @Override @@ -2564,6 +2560,19 @@ public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionLis public ClusterState execute(ClusterState currentState) throws Exception { final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final List snapshotEntries = findInProgressSnapshots(snapshots, snapshotNames, repoName); + boolean isSnapshotV2 = SHALLOW_SNAPSHOT_V2.get(repository.getMetadata().settings()); + boolean remoteStoreIndexShallowCopy = remoteStoreShallowCopyEnabled(repository); + List entriesForThisRepo = snapshots.entries() + .stream() + .filter(entry -> Objects.equals(entry.repository(), repoName)) + .collect(Collectors.toList()); + if (isSnapshotV2 && remoteStoreIndexShallowCopy && entriesForThisRepo.isEmpty() == false) { + throw new ConcurrentSnapshotExecutionException( + repoName, + String.join(",", snapshotNames), + "cannot delete snapshots in v2 repo while a snapshot is in progress" + ); + } final List snapshotIds = matchingSnapshotIds( snapshotEntries.stream().map(e -> e.snapshot().getSnapshotId()).collect(Collectors.toList()), repositoryData, From 15607b1ec44a8428f66c5d26c46b35fe1fc1bc50 Mon Sep 17 00:00:00 2001 From: SwethaGuptha <156877431+SwethaGuptha@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:15:33 +0530 Subject: [PATCH 278/357] Downgrade version to 2.18.0 for ser/de of new ClusterStatsRequest metric params. (#16441) Signed-off-by: Swetha Guptha Co-authored-by: Swetha Guptha --- .../test/java/org/opensearch/upgrades/ClusterStatsIT.java | 4 ++-- .../action/admin/cluster/stats/ClusterStatsRequest.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java index 1c5f35db8ec46..004bbd56bc526 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java @@ -26,8 +26,8 @@ public class ClusterStatsIT extends AbstractRollingTestCase { public void testClusterStats() throws IOException { Response response = client().performRequest(new Request("GET", "/_cluster/stats")); validateClusterStatsWithFilterResponse(response, nodeStatsMetrics, indicesStatsMetrics); - if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_3_0_0) || ( - CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_3_0_0))) { + if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_2_18_0) || ( + CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_2_18_0))) { response = client().performRequest(new Request("GET", "/_cluster/stats/os/nodes/_all")); validateClusterStatsWithFilterResponse(response, List.of("os"), Collections.emptyList()); response = client().performRequest(new Request("GET", "/_cluster/stats/indices/mappings/nodes/_all")); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java index 1c929881b898b..19b3033c9745b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -59,7 +59,7 @@ public ClusterStatsRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_16_0)) { useAggregatedNodeLevelResponses = in.readOptionalBoolean(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { computeAllMetrics = in.readOptionalBoolean(); final long longMetricsFlags = in.readLong(); for (Metric metric : Metric.values()) { @@ -135,7 +135,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_16_0)) { out.writeOptionalBoolean(useAggregatedNodeLevelResponses); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalBoolean(computeAllMetrics); long longMetricFlags = 0; for (Metric metric : requestedMetrics) { @@ -154,7 +154,7 @@ public void writeTo(StreamOutput out) throws IOException { * An enumeration of the "core" sections of metrics that may be requested * from the cluster stats endpoint. */ - @PublicApi(since = "3.0.0") + @PublicApi(since = "2.18.0") public enum Metric { OS("os", 0), JVM("jvm", 1), @@ -192,7 +192,7 @@ public int getIndex() { * * When no value is provided for param index_metric, default filter is set to _all. */ - @PublicApi(since = "3.0.0") + @PublicApi(since = "2.18.0") public enum IndexMetric { // Metrics computed from ShardStats SHARDS("shards", 0), From 5941a7e69dee8901b46f530551611a91b7de48c2 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 23 Oct 2024 20:41:34 +0800 Subject: [PATCH 279/357] Fix get index settings API doesn't show number_of_routing_shards when it was explicitly set on index creation (#16294) * Fix get index settings API doesn't show number_of_routing_shards when it was explicitly set on index creation Signed-off-by: Gao Binlong * Update skip version in rest yaml test file Signed-off-by: Gao Binlong * Fix test failure Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../40_number_of_routing_shards.yml | 41 +++++++++++++++++++ .../cluster/metadata/IndexMetadata.java | 3 +- .../metadata/MetadataCreateIndexService.java | 7 +--- .../MetadataCreateIndexServiceTests.java | 37 +++++++++++++++++ 5 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 332dad2a7370a..eb50efffcd52e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) - Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) +- Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294)) - Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) - [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) - Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml new file mode 100644 index 0000000000000..3fb392d6db134 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml @@ -0,0 +1,41 @@ +--- +setup: + - do: + indices.create: + body: + settings: + index: + number_of_routing_shards: 4 + number_of_shards: 2 + number_of_replicas: 1 + index: test-index + + - do: + indices.create: + body: + settings: + index: + number_of_shards: 2 + number_of_replicas: 1 + index: test-index1 + +--- +Test retrieval of number_routing_shards settings: + - skip: + version: " - 2.99.99" + reason: "introduced in 3.0.0" # TODO: change it to 2.18.0 after backport to 2.x branch + - do: + indices.get_settings: + flat_settings: true + index: test-index + # show `index.number_of_routing_shards` if it was explicitly set when creating + - match: + test-index.settings.index\.number_of_routing_shards: "4" + + - do: + indices.get_settings: + flat_settings: true + index: test-index1 + # do not show `index.number_of_routing_shards` if it was not explicitly set when creating + - match: + test-index1.settings.index\.number_of_routing_shards: null diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 713f8c9fc332c..c8ea5442a0dd0 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -300,7 +300,8 @@ public Iterator> settings() { } }, - Property.IndexScope + Property.IndexScope, + Property.NotCopyableOnResize ); /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index abda5dad25e4e..11df35527eea7 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -626,14 +626,9 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata( final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); final boolean isSystem = validateDotIndex(request.index(), isHiddenAfterTemplates); - // remove the setting it's temporary and is only relevant once we create the index - final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); - settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); - final Settings indexSettings = settingsBuilder.build(); - final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); - tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.settings(aggregatedIndexSettings); tmpImdBuilder.system(isSystem); addRemoteStoreCustomMetadata(tmpImdBuilder, true); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 1fdd038053eb6..0bb9ec28a1efc 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -136,6 +136,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; @@ -1821,6 +1822,42 @@ private void validateRemoteCustomData(Map customData, String exp assertEquals(expectedValue, customData.get(expectedKey)); } + public void testNumberOfRoutingShardsShowsInIndexSettings() { + withTemporaryClusterService(((clusterService, threadPool) -> { + MetadataCreateIndexService checkerService = new MetadataCreateIndexService( + Settings.EMPTY, + clusterService, + indicesServices, + null, + null, + createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService), + null, + null, + threadPool, + null, + new SystemIndices(Collections.emptyMap()), + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()), + DefaultRemoteStoreSettings.INSTANCE, + repositoriesServiceSupplier + ); + final int routingNumberOfShards = 4; + Settings indexSettings = Settings.builder() + .put("index.version.created", Version.CURRENT) + .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 2) + .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .put(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), routingNumberOfShards) + .build(); + CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + IndexMetadata indexMetadata = checkerService.buildAndValidateTemporaryIndexMetadata( + indexSettings, + request, + routingNumberOfShards + ); + assertEquals(INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(indexMetadata.getSettings()).intValue(), routingNumberOfShards); + })); + } + public void testGetIndexNumberOfRoutingShardsWithNullSourceIndex() { Settings indexSettings = Settings.builder() .put("index.version.created", Version.CURRENT) From 9a476b667244ca08530fc207d67af0add4821c43 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:50:11 +0530 Subject: [PATCH 280/357] Avoid making further transport calls if paginationStrategy outputs empty entities (#16444) Signed-off-by: Harsh Garg --- .../shards/TransportCatShardsAction.java | 10 ++++++++ .../indices/stats/IndicesStatsResponse.java | 5 ++++ .../rest/action/cat/RestIndicesAction.java | 24 +++++++++++++------ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java index 3dc8c38152a16..7b36b7a10f4f2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsAction.java @@ -108,6 +108,12 @@ public void onResponse(ClusterStateResponse clusterStateResponse) { : paginationStrategy.getRequestedEntities() ); catShardsResponse.setPageToken(Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken()); + // For paginated queries, if strategy outputs no shards to be returned, avoid fetching IndicesStats. + if (shouldSkipIndicesStatsRequest(paginationStrategy)) { + catShardsResponse.setIndicesStatsResponse(IndicesStatsResponse.getEmptyResponse()); + cancellableListener.onResponse(catShardsResponse); + return; + } IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); indicesStatsRequest.setShouldCancelOnTimeout(true); indicesStatsRequest.all(); @@ -159,4 +165,8 @@ private void validateRequestLimit( } } } + + private boolean shouldSkipIndicesStatsRequest(ShardPaginationStrategy paginationStrategy) { + return Objects.nonNull(paginationStrategy) && paginationStrategy.getRequestedEntities().isEmpty(); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java index ae989573b39ea..a5409e076730d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -45,6 +45,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -230,4 +231,8 @@ static final class Fields { public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this, true, false); } + + public static IndicesStatsResponse getEmptyResponse() { + return new IndicesStatsResponse(new ShardStats[0], 0, 0, 0, Collections.emptyList()); + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index b566ba9bbb8e4..ea73c474e90b8 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -212,13 +212,19 @@ public void onResponse(ClusterStateResponse clusterStateResponse) { groupedListener.onResponse(getSettingsResponse); groupedListener.onResponse(clusterStateResponse); - sendIndicesStatsRequest( - indicesToBeQueried, - subRequestIndicesOptions, - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); + // For paginated queries, if strategy outputs no indices to be returned, + // avoid fetching indices stats. + if (shouldSkipIndicesStatsRequest(paginationStrategy)) { + groupedListener.onResponse(IndicesStatsResponse.getEmptyResponse()); + } else { + sendIndicesStatsRequest( + indicesToBeQueried, + subRequestIndicesOptions, + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } sendClusterHealthRequest( indicesToBeQueried, @@ -1093,4 +1099,8 @@ public Tuple next() { }; } + private boolean shouldSkipIndicesStatsRequest(IndexPaginationStrategy paginationStrategy) { + return Objects.nonNull(paginationStrategy) && paginationStrategy.getRequestedEntities().isEmpty(); + } + } From 8eccbb5f78a7ffa18bedf3126134d6d877c42e17 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 23 Oct 2024 10:57:54 -0400 Subject: [PATCH 281/357] Add log message if SSL dual mode is enabled (#16437) * Add log message about dual mode enabled Signed-off-by: Craig Perkins * Add log message about dual mode enabled Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .../opensearch/transport/netty4/ssl/SecureNetty4Transport.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java index e51ed5663502f..90a9194d3cfd7 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java @@ -146,6 +146,7 @@ protected void initChannel(Channel ch) throws Exception { .map(SecureTransportSettingsProvider.SecureTransportParameters::dualModeEnabled) .orElse(false); if (dualModeEnabled) { + logger.info("SSL Dual mode enabled, using port unification handler"); final ChannelHandler portUnificationHandler = new DualModeSslHandler( settings, secureTransportSettingsProvider, From 66f01107d74d9c19d8646d1cf94cc79fe2350967 Mon Sep 17 00:00:00 2001 From: inpink <108166692+inpink@users.noreply.github.com> Date: Thu, 24 Oct 2024 00:54:06 +0900 Subject: [PATCH 282/357] Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic (#15807) (#16434) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated the test to account for Lucene's behavior where `IndexSearcher.search()` may return `GREATER_THAN_OR_EQUAL_TO` for totalHits when the number of matches exceeds 1000. - Added logic to check if `totalHits.relation` is `EQUAL_TO`. If so, assert that the count is exactly 11000. Otherwise, ensure the count is at least 11000 and within the allowed upper limit (`maxHits`). - This change prevents intermittent test failures caused by Lucene’s performance optimizations. Signed-off-by: inpink --- CHANGELOG.md | 1 + .../ApproximatePointRangeQueryTests.java | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb50efffcd52e..5062ffd830f41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) - Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) - [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422)) +- Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic ([#16434](https://github.com/opensearch-project/OpenSearch/pull/16434#pullrequestreview-2386999409)) ### Security diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index 9c022aade5dc6..4919cbc599892 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.opensearch.search.internal.SearchContext; @@ -175,6 +176,7 @@ public void testApproximateRangeWithSizeOverDefault() throws IOException { try { long lower = 0; long upper = 12000; + long maxHits = 12001; Query approximateQuery = new ApproximatePointRangeQuery( "point", pack(lower).bytes, @@ -188,7 +190,13 @@ protected String toString(int dimension, byte[] value) { }; IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(approximateQuery, 11000); - assertEquals(topDocs.totalHits, new TotalHits(11000, TotalHits.Relation.EQUAL_TO)); + + if (topDocs.totalHits.relation == Relation.EQUAL_TO) { + assertEquals(topDocs.totalHits.value, 11000); + } else { + assertTrue(11000 <= topDocs.totalHits.value); + assertTrue(maxHits >= topDocs.totalHits.value); + } } catch (IOException e) { throw new RuntimeException(e); } @@ -226,7 +234,7 @@ protected String toString(int dimension, byte[] value) { } }; Query query = LongPoint.newRangeQuery("point", lower, upper); - ; + IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(approximateQuery, 10); TopDocs topDocs1 = searcher.search(query, 10); @@ -235,7 +243,6 @@ protected String toString(int dimension, byte[] value) { assertNotEquals(topDocs.totalHits, topDocs1.totalHits); assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); assertEquals(topDocs1.totalHits, new TotalHits(101, TotalHits.Relation.EQUAL_TO)); - } catch (IOException e) { throw new RuntimeException(e); } @@ -278,7 +285,7 @@ protected String toString(int dimension, byte[] value) { } }; Query query = LongPoint.newRangeQuery("point", lower, upper); - ; + IndexSearcher searcher = new IndexSearcher(reader); Sort sort = new Sort(new SortField("point", SortField.Type.LONG)); TopDocs topDocs = searcher.search(approximateQuery, 10, sort); From 9dd1a59847ad8c2716d002716521ac40afc69355 Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Wed, 23 Oct 2024 12:16:57 -0700 Subject: [PATCH 283/357] Add support for restoring from snapshot with search replicas (#16111) * Add support for restoring from snapshot with search replicas Signed-off-by: Vinay Krishna Pudyodu * Fixed PR comments Signed-off-by: Vinay Krishna Pudyodu * Fixed the issues in gradle precommit Signed-off-by: Vinay Krishna Pudyodu * Fixed flaky tests Signed-off-by: Vinay Krishna Pudyodu * Fixed precommit failure Signed-off-by: Vinay Krishna Pudyodu * Fixed the codecoverage for RestoreService Signed-off-by: Vinay Krishna Pudyodu * Fixed precommit failure Signed-off-by: Vinay Krishna Pudyodu * removing unwanted tests Signed-off-by: Vinay Krishna Pudyodu * Update CHANGELOG.md Co-authored-by: Marc Handalian Signed-off-by: Vinay Krishna Pudyodu --------- Signed-off-by: Vinay Krishna Pudyodu Signed-off-by: Vinay Krishna Pudyodu Co-authored-by: Vinay Krishna Pudyodu Co-authored-by: Marc Handalian --- CHANGELOG.md | 2 + .../replication/SearchReplicaRestoreIT.java | 226 ++++++++++++++++++ .../cluster/routing/IndexRoutingTable.java | 11 + .../opensearch/snapshots/RestoreService.java | 36 +++ .../SearchOnlyReplicaRestoreTests.java | 59 +++++ .../snapshots/RestoreServiceTests.java | 37 +++ .../AbstractSnapshotIntegTestCase.java | 22 ++ 7 files changed, 393 insertions(+) create mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaRestoreIT.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/SearchOnlyReplicaRestoreTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5062ffd830f41..e682c1b226f4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292)) - Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)) - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) +- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) +- Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaRestoreIT.java new file mode 100644 index 0000000000000..352332b962c92 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaRestoreIT.java @@ -0,0 +1,226 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.indices.replication; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; +import org.opensearch.snapshots.SnapshotRestoreException; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SearchReplicaRestoreIT extends AbstractSnapshotIntegTestCase { + + private static final String INDEX_NAME = "test-idx-1"; + private static final String RESTORED_INDEX_NAME = INDEX_NAME + "-restored"; + private static final String REPOSITORY_NAME = "test-repo"; + private static final String SNAPSHOT_NAME = "test-snapshot"; + private static final String FS_REPOSITORY_TYPE = "fs"; + private static final int DOC_COUNT = 10; + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, true).build(); + } + + public void testSearchReplicaRestore_WhenSnapshotOnDocRep_RestoreOnDocRepWithSearchReplica() throws Exception { + bootstrapIndexWithOutSearchReplicas(ReplicationType.DOCUMENT); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build() + ) + ); + assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.DOCUMENT, ReplicationType.DOCUMENT))); + } + + public void testSearchReplicaRestore_WhenSnapshotOnDocRep_RestoreOnSegRepWithSearchReplica() throws Exception { + bootstrapIndexWithOutSearchReplicas(ReplicationType.DOCUMENT); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build() + ); + ensureYellowAndNoInitializingShards(RESTORED_INDEX_NAME); + internalCluster().startDataOnlyNode(); + ensureGreen(RESTORED_INDEX_NAME); + assertEquals(1, getNumberOfSearchReplicas(RESTORED_INDEX_NAME)); + + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testSearchReplicaRestore_WhenSnapshotOnSegRep_RestoreOnDocRepWithSearchReplica() throws Exception { + bootstrapIndexWithOutSearchReplicas(ReplicationType.SEGMENT); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build() + ) + ); + assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.SEGMENT, ReplicationType.DOCUMENT))); + } + + public void testSearchReplicaRestore_WhenSnapshotOnSegRep_RestoreOnSegRepWithSearchReplica() throws Exception { + bootstrapIndexWithOutSearchReplicas(ReplicationType.SEGMENT); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1).build() + ); + ensureYellowAndNoInitializingShards(RESTORED_INDEX_NAME); + internalCluster().startDataOnlyNode(); + ensureGreen(RESTORED_INDEX_NAME); + assertEquals(1, getNumberOfSearchReplicas(RESTORED_INDEX_NAME)); + + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + public void testSearchReplicaRestore_WhenSnapshotOnSegRepWithSearchReplica_RestoreOnDocRep() throws Exception { + bootstrapIndexWithSearchReplicas(); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build() + ) + ); + assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.SEGMENT, ReplicationType.DOCUMENT))); + } + + public void testSearchReplicaRestore_WhenSnapshotOnSegRepWithSearchReplica_RestoreOnDocRepWithNoSearchReplica() throws Exception { + bootstrapIndexWithSearchReplicas(); + createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME); + + restoreSnapshot( + REPOSITORY_NAME, + SNAPSHOT_NAME, + INDEX_NAME, + RESTORED_INDEX_NAME, + Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0) + .build() + ); + ensureGreen(RESTORED_INDEX_NAME); + assertEquals(0, getNumberOfSearchReplicas(RESTORED_INDEX_NAME)); + + SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); + assertHitCount(resp, DOC_COUNT); + } + + private void bootstrapIndexWithOutSearchReplicas(ReplicationType replicationType) throws InterruptedException { + startCluster(2); + + Settings settings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, replicationType) + .build(); + + createIndex(INDEX_NAME, settings); + indexRandomDocs(INDEX_NAME, DOC_COUNT); + refresh(INDEX_NAME); + ensureGreen(INDEX_NAME); + } + + private void bootstrapIndexWithSearchReplicas() throws InterruptedException { + startCluster(3); + + Settings settings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + + createIndex(INDEX_NAME, settings); + ensureGreen(INDEX_NAME); + for (int i = 0; i < DOC_COUNT; i++) { + client().prepareIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("foo", "bar").get(); + } + flushAndRefresh(INDEX_NAME); + } + + private void startCluster(int numOfNodes) { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(numOfNodes); + } + + private void createRepoAndSnapshot(String repositoryName, String repositoryType, String snapshotName, String indexName) { + createRepository(repositoryName, repositoryType, randomRepoPath().toAbsolutePath()); + createSnapshot(repositoryName, snapshotName, List.of(indexName)); + assertAcked(client().admin().indices().prepareDelete(INDEX_NAME)); + assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); + } + + private String getSnapshotExceptionMessage(ReplicationType snapshotReplicationType, ReplicationType restoreReplicationType) { + return "snapshot was created with [index.replication.type] as [" + + snapshotReplicationType + + "]. " + + "To restore with [index.replication.type] as [" + + restoreReplicationType + + "], " + + "[index.number_of_search_only_replicas] must be set to [0]"; + } + + private int getNumberOfSearchReplicas(String index) { + Metadata metadata = client().admin().cluster().prepareState().get().getState().metadata(); + return Integer.valueOf(metadata.index(index).getSettings().get(SETTING_NUMBER_OF_SEARCH_REPLICAS)); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index 9cc3bb21e2d12..b4592659bb70f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -573,6 +573,17 @@ private Builder initializeAsRestore( ); } } + for (int i = 0; i < indexMetadata.getNumberOfSearchOnlyReplicas(); i++) { + indexShardRoutingBuilder.addShard( + ShardRouting.newUnassigned( + shardId, + false, + true, + PeerRecoverySource.INSTANCE, // TODO: Update to remote store if enabled + unassignedInfo + ) + ); + } shards.put(shardNumber, indexShardRoutingBuilder.build()); } return this; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 88eff93e51b38..4b5bd951f80a0 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -92,6 +92,7 @@ import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.IndexId; @@ -120,10 +121,12 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; @@ -401,6 +404,13 @@ public ClusterState execute(ClusterState currentState) { overrideSettingsInternal, ignoreSettingsInternal ); + + validateReplicationTypeRestoreSettings( + snapshot, + metadata.index(index).getSettings().get(SETTING_REPLICATION_TYPE), + snapshotIndexMetadata + ); + if (isRemoteSnapshot) { snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId); } @@ -1303,6 +1313,32 @@ private static void validateSnapshotRestorable(final String repository, final Sn } } + // Visible for testing + static void validateReplicationTypeRestoreSettings(Snapshot snapshot, String snapshotReplicationType, IndexMetadata updatedMetadata) { + int restoreNumberOfSearchReplicas = updatedMetadata.getSettings().getAsInt(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0); + + if (restoreNumberOfSearchReplicas > 0 + && ReplicationType.DOCUMENT.toString().equals(updatedMetadata.getSettings().get(SETTING_REPLICATION_TYPE))) { + throw new SnapshotRestoreException( + snapshot, + "snapshot was created with [" + + SETTING_REPLICATION_TYPE + + "]" + + " as [" + + snapshotReplicationType + + "]." + + " To restore with [" + + SETTING_REPLICATION_TYPE + + "]" + + " as [" + + ReplicationType.DOCUMENT + + "], [" + + SETTING_NUMBER_OF_SEARCH_REPLICAS + + "] must be set to [0]" + ); + } + } + public static boolean failed(SnapshotInfo snapshot, String index) { for (SnapshotShardFailure failure : snapshot.shardFailures()) { if (index.equals(failure.index())) { diff --git a/server/src/test/java/org/opensearch/cluster/routing/SearchOnlyReplicaRestoreTests.java b/server/src/test/java/org/opensearch/cluster/routing/SearchOnlyReplicaRestoreTests.java new file mode 100644 index 0000000000000..d0effe9484533 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/SearchOnlyReplicaRestoreTests.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.IndexId; +import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashSet; + +public class SearchOnlyReplicaRestoreTests extends OpenSearchTestCase { + + public void testSearchOnlyReplicasRestored() { + Metadata metadata = Metadata.builder() + .put( + IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .numberOfSearchReplicas(1) + ) + .build(); + + IndexMetadata indexMetadata = metadata.index("test"); + RecoverySource.SnapshotRecoverySource snapshotRecoverySource = new RecoverySource.SnapshotRecoverySource( + UUIDs.randomBase64UUID(), + new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), + Version.CURRENT, + new IndexId("test", UUIDs.randomBase64UUID(random())) + ); + + RoutingTable routingTable = RoutingTable.builder().addAsNewRestore(indexMetadata, snapshotRecoverySource, new HashSet<>()).build(); + + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(routingTable) + .build(); + + IndexShardRoutingTable indexShardRoutingTable = clusterState.routingTable().index("test").shard(0); + + assertEquals(1, clusterState.routingTable().index("test").shards().size()); + assertEquals(3, indexShardRoutingTable.getShards().size()); + assertEquals(1, indexShardRoutingTable.searchOnlyReplicas().size()); + } +} diff --git a/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java index b56f1b47edc33..3629f324a62e4 100644 --- a/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RestoreServiceTests.java @@ -36,13 +36,17 @@ import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; import java.util.List; import static org.opensearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -122,4 +126,37 @@ public void testPrefixNotChanged() { assertEquals(renamedDataStreamName, renamedDataStream.getName()); assertEquals(Collections.singletonList(renamedIndex), renamedDataStream.getIndices()); } + + public void testValidateReplicationTypeRestoreSettings_WhenSnapshotIsDocument_RestoreToDocument() { + SnapshotId snapshotId = new SnapshotId("snapshotId", "123"); + Snapshot snapshot = new Snapshot("testRepo", snapshotId); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + Settings settings = Settings.builder() + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT.toString()) + .build(); + when(indexMetadata.getSettings()).thenReturn(settings); + + assertThrows( + SnapshotRestoreException.class, + () -> RestoreService.validateReplicationTypeRestoreSettings(snapshot, ReplicationType.DOCUMENT.toString(), indexMetadata) + ); + + } + + public void testValidateReplicationTypeRestoreSettings_WhenSnapshotIsSegment_RestoreToDocument() { + SnapshotId snapshotId = new SnapshotId("snapshotId", "123"); + Snapshot snapshot = new Snapshot("testRepo", snapshotId); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + Settings settings = Settings.builder() + .put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT.toString()) + .build(); + when(indexMetadata.getSettings()).thenReturn(settings); + + assertThrows( + SnapshotRestoreException.class, + () -> RestoreService.validateReplicationTypeRestoreSettings(snapshot, ReplicationType.SEGMENT.toString(), indexMetadata) + ); + } } diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 0bfa70a771f65..9bf725766069c 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -33,6 +33,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequest; @@ -61,6 +62,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.compress.CompressorRegistry; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -506,6 +508,26 @@ protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, Li return snapshotInfo; } + protected void restoreSnapshot( + String repositoryName, + String snapshotName, + String indexName, + String restoredIndexName, + Settings indexSettings + ) { + logger.info("--> restoring snapshot [{}] of {} in [{}] to [{}]", snapshotName, indexName, repositoryName, restoredIndexName); + RestoreSnapshotRequestBuilder builder = client().admin() + .cluster() + .prepareRestoreSnapshot(repositoryName, snapshotName) + .setWaitForCompletion(false) + .setRenamePattern(indexName) + .setRenameReplacement(restoredIndexName); + if (indexSettings != null) { + builder.setIndexSettings(indexSettings); + } + assertEquals(builder.get().status(), RestStatus.ACCEPTED); + } + protected void createIndexWithRandomDocs(String indexName, int docCount) throws InterruptedException { createIndex(indexName); ensureGreen(); From 119abaff95a7f54affea6c844fad7ce3c8360155 Mon Sep 17 00:00:00 2001 From: gargharsh3134 <51459091+gargharsh3134@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:00:55 +0530 Subject: [PATCH 284/357] Update version to 2.18.0 for CatShards request and response (#16455) * Update version to 2.18.0 for CatShards request and response Signed-off-by: Harsh Garg * Retry Build Signed-off-by: Harsh Garg * Fix CatShardsRequestTests Signed-off-by: Harsh Garg * Retry Build Signed-off-by: Harsh Garg --------- Signed-off-by: Harsh Garg Co-authored-by: Harsh Garg --- .../action/admin/cluster/shards/CatShardsRequest.java | 4 ++-- .../action/admin/cluster/shards/CatShardsResponse.java | 4 ++-- .../java/org/opensearch/action/pagination/PageParams.java | 2 +- .../action/admin/cluster/shards/CatShardsRequestTests.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java index 0319fa103138f..9511a45423e63 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java @@ -37,7 +37,7 @@ public CatShardsRequest() {} public CatShardsRequest(StreamInput in) throws IOException { super(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { indices = in.readStringArray(); cancelAfterTimeInterval = in.readOptionalTimeValue(); if (in.readBoolean()) { @@ -50,7 +50,7 @@ public CatShardsRequest(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { if (indices == null) { out.writeVInt(0); } else { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java index c2499ab190ded..dee0b0f2d45c0 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsResponse.java @@ -38,7 +38,7 @@ public CatShardsResponse() {} public CatShardsResponse(StreamInput in) throws IOException { super(in); indicesStatsResponse = new IndicesStatsResponse(in); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { nodes = DiscoveryNodes.readFrom(in, null); responseShards = in.readList(ShardRouting::new); if (in.readBoolean()) { @@ -50,7 +50,7 @@ public CatShardsResponse(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { indicesStatsResponse.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { nodes.writeToWithAttribute(out); out.writeList(responseShards); out.writeBoolean(pageToken != null); diff --git a/server/src/main/java/org/opensearch/action/pagination/PageParams.java b/server/src/main/java/org/opensearch/action/pagination/PageParams.java index 03de1aa465a15..6459893a8505f 100644 --- a/server/src/main/java/org/opensearch/action/pagination/PageParams.java +++ b/server/src/main/java/org/opensearch/action/pagination/PageParams.java @@ -20,7 +20,7 @@ * * Class specific to paginated queries, which will contain common query params required by a paginated API. */ -@PublicApi(since = "3.0.0") +@PublicApi(since = "2.18.0") public class PageParams implements Writeable { public static final String PARAM_SORT = "sort"; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java index f4215f54c1e21..e161342c3c609 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/shards/CatShardsRequestTests.java @@ -95,7 +95,7 @@ public void testSerializationWithOlderVersionsParametersNotSerialized() throws E catShardsRequest.setCancelAfterTimeInterval(TimeValue.timeValueMillis(randomIntBetween(1, 5))); catShardsRequest.setIndices(new String[2]); - Version version = VersionUtils.getPreviousVersion(Version.CURRENT); + Version version = VersionUtils.getPreviousVersion(Version.V_2_18_0); try (BytesStreamOutput out = new BytesStreamOutput()) { out.setVersion(version); catShardsRequest.writeTo(out); From 94987937d77f4aac65e5e57a56a3ab5f45b73bc5 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 24 Oct 2024 08:51:01 -0400 Subject: [PATCH 285/357] Update 2.x to 2.19.0 (since 2.18.0 was cut) (#16466) Signed-off-by: Andriy Redko --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 5587b8e5784c5..17bb5a7df9b21 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -41,3 +41,4 @@ BWC_VERSION: - "2.17.1" - "2.17.2" - "2.18.0" + - "2.19.0" \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 6b1f6898eec9b..4d685e3bc654a 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -113,6 +113,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); + public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { From 936cdb9043fb183cabe208fd7f7fc44a44b8f842 Mon Sep 17 00:00:00 2001 From: "Spencer G. Jones" Date: Thu, 24 Oct 2024 11:07:57 -0700 Subject: [PATCH 286/357] Change to correct version since this has been backported (#16472) Signed-off-by: Spencer G. Jones --- .../snapshots/restore/RestoreSnapshotRequest.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 42c64e04268e3..84bc87a5cb1ba 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -166,11 +166,10 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_17_0)) { sourceRemoteTranslogRepository = in.readOptionalString(); } - // TODO: change to V_2_18_0 once this is backported into that version - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { renameAliasPattern = in.readOptionalString(); } - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_18_0)) { renameAliasReplacement = in.readOptionalString(); } } @@ -200,11 +199,10 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeOptionalString(sourceRemoteTranslogRepository); } - // TODO: change to V_2_18_0 once this is backported into that version - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalString(renameAliasPattern); } - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalString(renameAliasReplacement); } } From 4ad1be3825f8a47ff94fdb3cee63db78c2c142a7 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:08:05 +0530 Subject: [PATCH 287/357] Revert uploading of manifest using min codec version (#16403) Signed-off-by: Sooraj Sinha --- CHANGELOG.md | 1 + .../PublicationTransportHandler.java | 1 - .../opensearch/gateway/GatewayMetaState.java | 14 +-- .../remote/RemoteClusterStateService.java | 12 +-- .../gateway/remote/RemoteManifestManager.java | 5 +- .../coordination/CoordinationStateTests.java | 6 +- .../GatewayMetaStatePersistedStateTests.java | 36 +++---- .../RemoteClusterStateServiceTests.java | 97 ++++++------------- 8 files changed, 57 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e682c1b226f4a..f5dc69a9ec290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) - [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422)) - Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic ([#16434](https://github.com/opensearch-project/OpenSearch/pull/16434#pullrequestreview-2386999409)) +- Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403)) ### Security diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index d30efde52bffb..c4cb484cda693 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -367,7 +367,6 @@ public PublicationContext newPublicationContext( } private boolean validateRemotePublicationConfiguredOnAllNodes(DiscoveryNodes discoveryNodes) { - assert ClusterMetadataManifest.getCodecForVersion(discoveryNodes.getMinNodeVersion()) >= ClusterMetadataManifest.CODEC_V0; for (DiscoveryNode node : discoveryNodes.getNodes().values()) { // if a node is non-remote then created local publication context if (node.isRemoteStatePublicationEnabled() == false) { diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index f5da6df2689bd..1e2f5612ca002 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -753,12 +753,8 @@ public void setLastAcceptedState(ClusterState clusterState) { } try { final RemoteClusterStateManifestInfo manifestDetails; - // Decide the codec version - int codecVersion = ClusterMetadataManifest.getCodecForVersion(clusterState.nodes().getMinNodeVersion()); - assert codecVersion >= 0 : codecVersion; - logger.info("codec version is {}", codecVersion); - if (shouldWriteFullClusterState(clusterState, codecVersion)) { + if (shouldWriteFullClusterState(clusterState)) { final Optional latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID() @@ -775,7 +771,7 @@ public void setLastAcceptedState(ClusterState clusterState) { clusterState.metadata().clusterUUID() ); } - manifestDetails = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID, codecVersion); + manifestDetails = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID); } else { assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == true : "Previous manifest and previous ClusterState are not in sync"; @@ -820,13 +816,11 @@ private boolean verifyManifestAndClusterState(ClusterMetadataManifest manifest, return true; } - private boolean shouldWriteFullClusterState(ClusterState clusterState, int codecVersion) { - assert lastAcceptedManifest == null || lastAcceptedManifest.getCodecVersion() <= codecVersion; + private boolean shouldWriteFullClusterState(ClusterState clusterState) { if (lastAcceptedState == null || lastAcceptedManifest == null || (remoteClusterStateService.isRemotePublicationEnabled() == false && lastAcceptedState.term() != clusterState.term()) - || lastAcceptedManifest.getOpensearchVersion() != Version.CURRENT - || lastAcceptedManifest.getCodecVersion() != codecVersion) { + || lastAcceptedManifest.getOpensearchVersion() != Version.CURRENT) { return true; } return false; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 0cd2025b98783..dc41189afc3cb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -301,8 +301,7 @@ public RemoteClusterStateService( * @return A manifest object which contains the details of uploaded entity metadata. */ @Nullable - public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterState, String previousClusterUUID, int codecVersion) - throws IOException { + public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterState, String previousClusterUUID) throws IOException { final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); @@ -342,8 +341,7 @@ public RemoteClusterStateManifestInfo writeFullMetadata(ClusterState clusterStat !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState, threadpool) : null, - false, - codecVersion + false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -551,8 +549,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState, threadpool) : null, - false, - previousManifest.getCodecVersion() + false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -1024,8 +1021,7 @@ public RemoteClusterStateManifestInfo markLastStateAsCommitted( !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) ? new ClusterStateChecksum(clusterState, threadpool) : null, - true, - previousManifest.getCodecVersion() + true ); if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.getClusterMetadataManifest().isClusterUUIDCommitted()) { remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifestDetails.getClusterMetadataManifest()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index b243269fe323e..20e14ff805ca8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -100,8 +100,7 @@ RemoteClusterStateManifestInfo uploadManifest( String previousClusterUUID, ClusterStateDiffManifest clusterDiffManifest, ClusterStateChecksum clusterStateChecksum, - boolean committed, - int codecVersion + boolean committed ) { synchronized (this) { ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); @@ -112,7 +111,7 @@ RemoteClusterStateManifestInfo uploadManifest( .opensearchVersion(Version.CURRENT) .nodeId(nodeId) .committed(committed) - .codecVersion(codecVersion) + .codecVersion(ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION) .indices(uploadedMetadataResult.uploadedIndexMetadata) .previousClusterUUID(previousClusterUUID) .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index b5d16e7be849f..f707198efb073 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -67,7 +67,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_PUBLICATION_SETTING_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -962,7 +961,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep .previousClusterUUID(randomAlphaOfLength(10)) .clusterUUIDCommitted(true) .build(); - when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION)).thenReturn( + when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn( new RemoteClusterStateManifestInfo(manifest, "path/to/manifest") ); @@ -975,8 +974,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep final CoordinationState coordinationState = createCoordinationState(persistedStateRegistry, node1, remoteStateSettings()); coordinationState.handlePrePublish(clusterState); - Mockito.verify(remoteClusterStateService, Mockito.times(1)) - .writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService, Mockito.times(1)).writeFullMetadata(clusterState, previousClusterUUID); assertThat(persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE).getLastAcceptedState(), equalTo(clusterState)); when(remoteClusterStateService.markLastStateAsCommitted(any(), any(), eq(false))).thenReturn( diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index efdb3076f419c..955ea82e219e8 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -759,7 +759,7 @@ public void testRemotePersistedState() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder().clusterTerm(1L).stateVersion(5L).build(); final String previousClusterUUID = "prev-cluster-uuid"; - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION))) + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())) .thenReturn(new RemoteClusterStateManifestInfo(manifest, "path/to/manifest")); Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())) @@ -777,7 +777,7 @@ public void testRemotePersistedState() throws IOException { ); remotePersistedState.setLastAcceptedState(clusterState); - Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), equalTo(clusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); @@ -789,8 +789,7 @@ public void testRemotePersistedState() throws IOException { ); remotePersistedState.setLastAcceptedState(secondClusterState); - Mockito.verify(remoteClusterStateService, times(1)) - .writeFullMetadata(secondClusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService, times(1)).writeFullMetadata(secondClusterState, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), equalTo(secondClusterState)); assertThat(remotePersistedState.getCurrentTerm(), equalTo(clusterTerm)); @@ -820,9 +819,9 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx .clusterTerm(1L) .stateVersion(5L) .codecVersion(CODEC_V1) - .opensearchVersion(Version.CURRENT) + .opensearchVersion(Version.V_2_15_0) .build(); - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any(), eq(CODEC_V1))) + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())) .thenReturn(new RemoteClusterStateManifestInfo(manifest, "path/to/manifest2")); CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); @@ -833,7 +832,7 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx ); remotePersistedState.setLastAcceptedState(clusterState1); - Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState1, previousClusterUUID, CODEC_V1); + Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState1, previousClusterUUID); ClusterState clusterState2 = createClusterState( randomNonNegativeLong(), @@ -846,10 +845,10 @@ public void testRemotePersistedStateWithDifferentNodeConfiguration() throws IOEx .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) .opensearchVersion(Version.CURRENT) .build(); - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION))) + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())) .thenReturn(new RemoteClusterStateManifestInfo(manifest2, "path/to/manifest")); remotePersistedState.setLastAcceptedState(clusterState2); - Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState2, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterState2, previousClusterUUID); ClusterState clusterState3 = createClusterState( randomNonNegativeLong(), @@ -889,8 +888,7 @@ public void testRemotePersistentState_FollowerNode() throws IOException { remotePersistedState.setLastAcceptedState(clusterState); remotePersistedState.setLastAcceptedManifest(manifest); - Mockito.verify(remoteClusterStateService, never()) - .writeFullMetadata(clusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService, never()).writeFullMetadata(clusterState, previousClusterUUID); assertEquals(clusterState, remotePersistedState.getLastAcceptedState()); assertEquals(clusterTerm, remotePersistedState.getCurrentTerm()); @@ -906,8 +904,7 @@ public void testRemotePersistentState_FollowerNode() throws IOException { ); remotePersistedState.setLastAcceptedState(secondClusterState); - Mockito.verify(remoteClusterStateService, never()) - .writeFullMetadata(secondClusterState, previousClusterUUID, MANIFEST_CURRENT_CODEC_VERSION); + Mockito.verify(remoteClusterStateService, never()).writeFullMetadata(secondClusterState, previousClusterUUID); assertEquals(secondClusterState, remotePersistedState.getLastAcceptedState()); assertEquals(clusterTerm, remotePersistedState.getCurrentTerm()); @@ -940,7 +937,7 @@ public void testRemotePersistedStateNotCommitted() throws IOException { .build(); Mockito.when(remoteClusterStateService.getLatestClusterMetadataManifest(Mockito.any(), Mockito.any())) .thenReturn(Optional.of(manifest)); - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION))) + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())) .thenReturn(new RemoteClusterStateManifestInfo(manifest, "path/to/manifest")); Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())) @@ -966,17 +963,14 @@ public void testRemotePersistedStateNotCommitted() throws IOException { remotePersistedState.setLastAcceptedState(clusterState); ArgumentCaptor previousClusterUUIDCaptor = ArgumentCaptor.forClass(String.class); ArgumentCaptor clusterStateCaptor = ArgumentCaptor.forClass(ClusterState.class); - Mockito.verify(remoteClusterStateService) - .writeFullMetadata(clusterStateCaptor.capture(), previousClusterUUIDCaptor.capture(), eq(MANIFEST_CURRENT_CODEC_VERSION)); + Mockito.verify(remoteClusterStateService).writeFullMetadata(clusterStateCaptor.capture(), previousClusterUUIDCaptor.capture()); assertEquals(previousClusterUUID, previousClusterUUIDCaptor.getValue()); } public void testRemotePersistedStateExceptionOnFullStateUpload() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final String previousClusterUUID = "prev-cluster-uuid"; - Mockito.doThrow(IOException.class) - .when(remoteClusterStateService) - .writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION)); + Mockito.doThrow(IOException.class).when(remoteClusterStateService).writeFullMetadata(Mockito.any(), Mockito.any()); CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); @@ -994,9 +988,7 @@ public void testRemotePersistedStateFailureStats() throws IOException { RemoteUploadStats remoteStateStats = new RemoteUploadStats(); final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final String previousClusterUUID = "prev-cluster-uuid"; - Mockito.doThrow(IOException.class) - .when(remoteClusterStateService) - .writeFullMetadata(Mockito.any(), Mockito.any(), eq(MANIFEST_CURRENT_CODEC_VERSION)); + Mockito.doThrow(IOException.class).when(remoteClusterStateService).writeFullMetadata(Mockito.any(), Mockito.any()); when(remoteClusterStateService.getUploadStats()).thenReturn(remoteStateStats); doAnswer((i) -> { remoteStateStats.stateFailed(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index dffbb9d82545a..448b9cc9d78ac 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -280,8 +280,7 @@ public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException final ClusterState clusterState = generateClusterStateWithOneIndex().build(); final RemoteClusterStateManifestInfo manifestDetails = remoteClusterStateService.writeFullMetadata( clusterState, - randomAlphaOfLength(10), - MANIFEST_CURRENT_CODEC_VERSION + randomAlphaOfLength(10) ); Assert.assertThat(manifestDetails, nullValue()); } @@ -327,11 +326,8 @@ public void testWriteFullMetadataSuccess() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -401,11 +397,8 @@ public void testWriteFullMetadataSuccessPublicationEnabled() throws IOException .build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -453,11 +446,8 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -534,7 +524,7 @@ public void run() { remoteClusterStateService.start(); assertThrows( RemoteStateTransferException.class, - () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) + () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); } @@ -578,7 +568,7 @@ public void testTimeoutWhileWritingManifestFile() throws IOException { ).thenReturn(new RemoteClusterStateUtils.UploadedMetadataResults()); RemoteStateTransferException ex = expectThrows( RemoteStateTransferException.class, - () -> spiedService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) + () -> spiedService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); assertTrue(ex.getMessage().contains("Timed out waiting for transfer")); } @@ -600,7 +590,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx remoteClusterStateService.start(); assertThrows( RemoteStateTransferException.class, - () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10), MANIFEST_CURRENT_CODEC_VERSION) + () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); assertEquals(0, remoteClusterStateService.getUploadStats().getSuccessCount()); } @@ -1850,7 +1840,7 @@ private void verifyCodecMigrationManifest(int previousCodec) throws IOException // global metadata is updated assertThat(manifestAfterUpdate.hasMetadataAttributesFiles(), is(true)); // During incremental update, codec version will not change. - assertThat(manifestAfterUpdate.getCodecVersion(), is(previousCodec)); + assertThat(manifestAfterUpdate.getCodecVersion(), is(MANIFEST_CURRENT_CODEC_VERSION)); } public void testWriteIncrementalGlobalMetadataFromCodecV0Success() throws IOException { @@ -1885,7 +1875,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe ).getClusterMetadataManifest(); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .codecVersion(previousManifest.getCodecVersion()) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) .indices(Collections.emptyList()) .clusterTerm(1L) .stateVersion(1L) @@ -2074,11 +2064,8 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata( - initialClusterState, - "_na_", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); ClusterState clusterState1 = ClusterState.builder(initialClusterState) .metadata( @@ -2156,11 +2143,8 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata( - initialClusterState, - "_na_", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); String initialIndex = "test-index"; Index index1 = new Index("test-index-1", "index-uuid-1"); Index index2 = new Index("test-index-2", "index-uuid-2"); @@ -2238,11 +2222,8 @@ private void verifyMetadataAttributeOnlyUpdated( // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata( - initialClusterState, - "_na_", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); ClusterState newClusterState = clusterStateUpdater.apply(initialClusterState); @@ -2255,11 +2236,8 @@ private void verifyMetadataAttributeOnlyUpdated( initialManifest ).getClusterMetadataManifest(); } else { - manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata( - newClusterState, - initialClusterState.stateUUID(), - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata(newClusterState, initialClusterState.stateUUID()) + .getClusterMetadataManifest(); } assertions.accept(initialManifest, manifestAfterMetadataUpdate); @@ -2742,11 +2720,8 @@ public void testRemoteStateUploadStats() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); assertTrue(remoteClusterStateService.getUploadStats() != null); assertEquals(1, remoteClusterStateService.getUploadStats().getSuccessCount()); @@ -2801,11 +2776,8 @@ public void testWriteFullMetadataSuccessWithRoutingTable() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index", @@ -2854,11 +2826,8 @@ public void testWriteFullMetadataInParallelSuccessWithRoutingTable() throws IOEx when((blobStoreRepository.basePath())).thenReturn(BlobPath.cleanPath().add("base-path")); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( @@ -3108,11 +3077,8 @@ public void testWriteFullMetadataSuccessWithChecksumValidationEnabled() throws I final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index", @@ -3152,11 +3118,8 @@ public void testWriteFullMetadataSuccessWithChecksumValidationModeNone() throws final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata( - clusterState, - "prev-cluster-uuid", - MANIFEST_CURRENT_CODEC_VERSION - ).getClusterMetadataManifest(); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final UploadedIndexMetadata uploadedIndiceRoutingMetadata = new UploadedIndexMetadata( "test-index", From bb45f0343553944627b18a3fd8fa0b79d0991b69 Mon Sep 17 00:00:00 2001 From: Arpit-Bandejiya Date: Fri, 25 Oct 2024 17:46:59 +0530 Subject: [PATCH 288/357] Add Setting to adjust the primary constraint weights (#16471) Add Setting to adjust the primary constraint weights (#16471) Signed-off-by: Arpit Bandejiya --- CHANGELOG.md | 1 + .../allocation/AllocationConstraints.java | 4 +-- .../routing/allocation/Constraint.java | 13 ++++--- .../routing/allocation/ConstraintTypes.java | 10 ++++++ .../allocation/RebalanceConstraints.java | 4 +-- .../allocator/BalancedShardsAllocator.java | 35 ++++++++++++++++--- .../common/settings/ClusterSettings.java | 1 + .../AllocationConstraintsTests.java | 28 ++++++++------- 8 files changed, 71 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5dc69a9ec290..d6e76d946866b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) - [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) +- Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java index 6702db4b43e91..4b71347e4071b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java @@ -39,8 +39,8 @@ public void updateAllocationConstraint(String constraint, boolean enable) { this.constraints.get(constraint).setEnable(enable); } - public long weight(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index) { - Constraint.ConstraintParams params = new Constraint.ConstraintParams(balancer, node, index); + public long weight(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index, long primaryThresholdWeight) { + Constraint.ConstraintParams params = new Constraint.ConstraintParams(balancer, node, index, primaryThresholdWeight); return params.weight(constraints); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/Constraint.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/Constraint.java index e9c3c0afcbe88..ce0bb70d7d0b7 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/Constraint.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/Constraint.java @@ -14,7 +14,7 @@ import java.util.Map; import java.util.function.Predicate; -import static org.opensearch.cluster.routing.allocation.ConstraintTypes.CONSTRAINT_WEIGHT; +import static org.opensearch.cluster.routing.allocation.ConstraintTypes.predicateKeyToWeightMap; /** * Defines a constraint useful to de-prioritize certain nodes as target of unassigned shards used in {@link AllocationConstraints} or @@ -44,11 +44,13 @@ static class ConstraintParams { private ShardsBalancer balancer; private BalancedShardsAllocator.ModelNode node; private String index; + private long PrimaryConstraintThreshold; - ConstraintParams(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index) { + ConstraintParams(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index, long primaryConstraintThreshold) { this.balancer = balancer; this.node = node; this.index = index; + this.PrimaryConstraintThreshold = primaryConstraintThreshold; } public ShardsBalancer getBalancer() { @@ -75,9 +77,12 @@ public String getIndex() { */ public long weight(Map constraints) { long totalConstraintWeight = 0; - for (Constraint constraint : constraints.values()) { + for (Map.Entry entry : constraints.entrySet()) { + String key = entry.getKey(); + Constraint constraint = entry.getValue(); if (constraint.test(this)) { - totalConstraintWeight += CONSTRAINT_WEIGHT; + double weight = predicateKeyToWeightMap(key, PrimaryConstraintThreshold); + totalConstraintWeight += weight; } } return totalConstraintWeight; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java index 28ad199218884..ff40556a4894b 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java @@ -86,4 +86,14 @@ public static Predicate isPrimaryShardsPerNodeBreac return primaryShardCount >= allowedPrimaryShardCount; }; } + + public static long predicateKeyToWeightMap(String key, long primaryConstraintWeight) { + switch (key) { + case CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID: + case CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID: + return primaryConstraintWeight; + default: + return CONSTRAINT_WEIGHT; + } + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java index 2c2138af18abc..803ef04ce1675 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java @@ -42,8 +42,8 @@ public void updateRebalanceConstraint(String constraint, boolean enable) { this.constraints.get(constraint).setEnable(enable); } - public long weight(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index) { - Constraint.ConstraintParams params = new Constraint.ConstraintParams(balancer, node, index); + public long weight(ShardsBalancer balancer, BalancedShardsAllocator.ModelNode node, String index, long primaryConstraintThreshold) { + Constraint.ConstraintParams params = new Constraint.ConstraintParams(balancer, node, index, primaryConstraintThreshold); return params.weight(constraints); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 785636fa7ff2a..cfbb4d34c3a38 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -139,6 +139,14 @@ public class BalancedShardsAllocator implements ShardsAllocator { Property.NodeScope ); + public static final Setting PRIMARY_CONSTRAINT_THRESHOLD_SETTING = Setting.longSetting( + "cluster.routing.allocation.primary_constraint.threshold", + 10, + 0, + Property.Dynamic, + Property.NodeScope + ); + /** * This setting governs whether primary shards balance is desired during allocation. This is used by {@link ConstraintTypes#isPerIndexPrimaryShardsPerNodeBreached()} * and {@link ConstraintTypes#isPrimaryShardsPerNodeBreached} which is used during unassigned shard allocation @@ -201,6 +209,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { private volatile float shardBalanceFactor; private volatile WeightFunction weightFunction; private volatile float threshold; + private volatile long primaryConstraintThreshold; private volatile boolean ignoreThrottleInRestore; private volatile TimeValue allocatorTimeout; @@ -219,6 +228,7 @@ public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSetting setIgnoreThrottleInRestore(IGNORE_THROTTLE_FOR_REMOTE_RESTORE.get(settings)); updateWeightFunction(); setThreshold(THRESHOLD_SETTING.get(settings)); + setPrimaryConstraintThresholdSetting(PRIMARY_CONSTRAINT_THRESHOLD_SETTING.get(settings)); setPreferPrimaryShardBalance(PREFER_PRIMARY_SHARD_BALANCE.get(settings)); setPreferPrimaryShardRebalance(PREFER_PRIMARY_SHARD_REBALANCE.get(settings)); setShardMovementStrategy(SHARD_MOVEMENT_STRATEGY_SETTING.get(settings)); @@ -231,6 +241,7 @@ public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSetting clusterSettings.addSettingsUpdateConsumer(PRIMARY_SHARD_REBALANCE_BUFFER, this::updatePreferPrimaryShardBalanceBuffer); clusterSettings.addSettingsUpdateConsumer(PREFER_PRIMARY_SHARD_REBALANCE, this::setPreferPrimaryShardRebalance); clusterSettings.addSettingsUpdateConsumer(THRESHOLD_SETTING, this::setThreshold); + clusterSettings.addSettingsUpdateConsumer(PRIMARY_CONSTRAINT_THRESHOLD_SETTING, this::setPrimaryConstraintThresholdSetting); clusterSettings.addSettingsUpdateConsumer(IGNORE_THROTTLE_FOR_REMOTE_RESTORE, this::setIgnoreThrottleInRestore); clusterSettings.addSettingsUpdateConsumer(ALLOCATOR_TIMEOUT_SETTING, this::setAllocatorTimeout); } @@ -294,7 +305,12 @@ private void updatePreferPrimaryShardBalanceBuffer(float preferPrimaryShardBalan } private void updateWeightFunction() { - weightFunction = new WeightFunction(this.indexBalanceFactor, this.shardBalanceFactor, this.preferPrimaryShardRebalanceBuffer); + weightFunction = new WeightFunction( + this.indexBalanceFactor, + this.shardBalanceFactor, + this.preferPrimaryShardRebalanceBuffer, + this.primaryConstraintThreshold + ); } /** @@ -317,6 +333,11 @@ private void setThreshold(float threshold) { this.threshold = threshold; } + private void setPrimaryConstraintThresholdSetting(long threshold) { + this.primaryConstraintThreshold = threshold; + this.weightFunction.updatePrimaryConstraintThreshold(threshold); + } + private void setAllocatorTimeout(TimeValue allocatorTimeout) { this.allocatorTimeout = allocatorTimeout; } @@ -489,10 +510,11 @@ static class WeightFunction { private final float shardBalance; private final float theta0; private final float theta1; + private long primaryConstraintThreshold; private AllocationConstraints constraints; private RebalanceConstraints rebalanceConstraints; - WeightFunction(float indexBalance, float shardBalance, float preferPrimaryBalanceBuffer) { + WeightFunction(float indexBalance, float shardBalance, float preferPrimaryBalanceBuffer, long primaryConstraintThreshold) { float sum = indexBalance + shardBalance; if (sum <= 0.0f) { throw new IllegalArgumentException("Balance factors must sum to a value > 0 but was: " + sum); @@ -501,6 +523,7 @@ static class WeightFunction { theta1 = indexBalance / sum; this.indexBalance = indexBalance; this.shardBalance = shardBalance; + this.primaryConstraintThreshold = primaryConstraintThreshold; RebalanceParameter rebalanceParameter = new RebalanceParameter(preferPrimaryBalanceBuffer); this.constraints = new AllocationConstraints(); this.rebalanceConstraints = new RebalanceConstraints(rebalanceParameter); @@ -510,12 +533,12 @@ static class WeightFunction { public float weightWithAllocationConstraints(ShardsBalancer balancer, ModelNode node, String index) { float balancerWeight = weight(balancer, node, index); - return balancerWeight + constraints.weight(balancer, node, index); + return balancerWeight + constraints.weight(balancer, node, index, primaryConstraintThreshold); } public float weightWithRebalanceConstraints(ShardsBalancer balancer, ModelNode node, String index) { float balancerWeight = weight(balancer, node, index); - return balancerWeight + rebalanceConstraints.weight(balancer, node, index); + return balancerWeight + rebalanceConstraints.weight(balancer, node, index, primaryConstraintThreshold); } float weight(ShardsBalancer balancer, ModelNode node, String index) { @@ -531,6 +554,10 @@ void updateAllocationConstraint(String constraint, boolean enable) { void updateRebalanceConstraint(String constraint, boolean add) { this.rebalanceConstraints.updateRebalanceConstraint(constraint, add); } + + void updatePrimaryConstraintThreshold(long primaryConstraintThreshold) { + this.primaryConstraintThreshold = primaryConstraintThreshold; + } } /** diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index f769f8729c25b..c1f4e52706465 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -276,6 +276,7 @@ public void apply(Settings value, Settings current, Settings previous) { BalancedShardsAllocator.THRESHOLD_SETTING, BalancedShardsAllocator.IGNORE_THROTTLE_FOR_REMOTE_RESTORE, BalancedShardsAllocator.ALLOCATOR_TIMEOUT_SETTING, + BalancedShardsAllocator.PRIMARY_CONSTRAINT_THRESHOLD_SETTING, BreakerSettings.CIRCUIT_BREAKER_LIMIT_SETTING, BreakerSettings.CIRCUIT_BREAKER_OVERHEAD_SETTING, BreakerSettings.CIRCUIT_BREAKER_TYPE, diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationConstraintsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationConstraintsTests.java index 4c9fcd1650664..23d0b708441f8 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationConstraintsTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationConstraintsTests.java @@ -25,6 +25,8 @@ public class AllocationConstraintsTests extends OpenSearchAllocationTestCase { + long constraintWeight = 20L; + public void testSettings() { Settings.Builder settings = Settings.builder(); ClusterSettings service = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); @@ -69,7 +71,7 @@ public void testIndexShardsPerNodeConstraint() { when(node.getNodeId()).thenReturn("test-node"); long expectedWeight = (shardCount >= avgShardsPerNode) ? CONSTRAINT_WEIGHT : 0; - assertEquals(expectedWeight, constraints.weight(balancer, node, "index")); + assertEquals(expectedWeight, constraints.weight(balancer, node, "index", constraintWeight)); } @@ -91,14 +93,14 @@ public void testPerIndexPrimaryShardsConstraint() { when(node.numPrimaryShards(anyString())).thenReturn(perIndexPrimaryShardCount); when(node.getNodeId()).thenReturn("test-node"); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); perIndexPrimaryShardCount = 2; when(node.numPrimaryShards(anyString())).thenReturn(perIndexPrimaryShardCount); - assertEquals(CONSTRAINT_WEIGHT, constraints.weight(balancer, node, indexName)); + assertEquals(CONSTRAINT_WEIGHT, constraints.weight(balancer, node, indexName, constraintWeight)); constraints.updateAllocationConstraint(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, false); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); } /** @@ -118,14 +120,14 @@ public void testGlobalPrimaryShardsConstraint() { when(node.numPrimaryShards()).thenReturn(primaryShardCount); when(node.getNodeId()).thenReturn("test-node"); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); primaryShardCount = 3; when(node.numPrimaryShards()).thenReturn(primaryShardCount); - assertEquals(CONSTRAINT_WEIGHT, constraints.weight(balancer, node, indexName)); + assertEquals(constraintWeight, constraints.weight(balancer, node, indexName, constraintWeight)); constraints.updateAllocationConstraint(CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, false); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); } /** @@ -150,22 +152,22 @@ public void testPrimaryShardsConstraints() { when(node.numPrimaryShards()).thenReturn(primaryShardCount); when(node.getNodeId()).thenReturn("test-node"); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); // breaching global primary shard count but not per index primary shard count primaryShardCount = 5; when(node.numPrimaryShards()).thenReturn(primaryShardCount); - assertEquals(CONSTRAINT_WEIGHT, constraints.weight(balancer, node, indexName)); + assertEquals(constraintWeight, constraints.weight(balancer, node, indexName, constraintWeight)); // when per index primary shard count constraint is also breached perIndexPrimaryShardCount = 3; when(node.numPrimaryShards(indexName)).thenReturn(perIndexPrimaryShardCount); - assertEquals(2 * CONSTRAINT_WEIGHT, constraints.weight(balancer, node, indexName)); + assertEquals(CONSTRAINT_WEIGHT + constraintWeight, constraints.weight(balancer, node, indexName, constraintWeight)); // disable both constraints constraints.updateAllocationConstraint(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, false); constraints.updateAllocationConstraint(CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, false); - assertEquals(0, constraints.weight(balancer, node, indexName)); + assertEquals(0, constraints.weight(balancer, node, indexName, constraintWeight)); } /** @@ -202,8 +204,8 @@ public void testAllConstraints() { long expectedWeight = (shardCount >= (int) Math.ceil(avgPerIndexShardsPerNode)) ? CONSTRAINT_WEIGHT : 0; expectedWeight += perIndexPrimaryShardCount > (int) Math.ceil(avgPerIndexPrimaryShardsPerNode) ? CONSTRAINT_WEIGHT : 0; - expectedWeight += primaryShardsPerNode >= (int) Math.ceil(avgPrimaryShardsPerNode) ? CONSTRAINT_WEIGHT : 0; - assertEquals(expectedWeight, constraints.weight(balancer, node, indexName)); + expectedWeight += primaryShardsPerNode >= (int) Math.ceil(avgPrimaryShardsPerNode) ? constraintWeight : 0; + assertEquals(expectedWeight, constraints.weight(balancer, node, indexName, constraintWeight)); } } From b2d537a01a368e11840582b656264f69cb06ed4d Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Sat, 26 Oct 2024 00:02:58 +0800 Subject: [PATCH 289/357] Update version check in yaml test file for the bug fix for get index settings API (#16480) Signed-off-by: Gao Binlong --- .../test/indices.get_settings/40_number_of_routing_shards.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml index 3fb392d6db134..3c86678a690b8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_settings/40_number_of_routing_shards.yml @@ -22,8 +22,8 @@ setup: --- Test retrieval of number_routing_shards settings: - skip: - version: " - 2.99.99" - reason: "introduced in 3.0.0" # TODO: change it to 2.18.0 after backport to 2.x branch + version: " - 2.18.99" + reason: "introduced in 2.19.0" - do: indices.get_settings: flat_settings: true From 6f1b59e54bec41d40772f8571c7b65d4b523f8b1 Mon Sep 17 00:00:00 2001 From: Sumit Bansal Date: Sat, 26 Oct 2024 11:34:44 +0530 Subject: [PATCH 290/357] Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. (#16421) Signed-off-by: Sumit Bansal Signed-off-by: shwetathareja Co-authored-by: shwetathareja --- CHANGELOG.md | 1 + .../cluster/service/MasterService.java | 32 ++++++++------ .../cluster/service/TaskBatcher.java | 34 +++++++++----- .../cluster/service/MasterServiceTests.java | 44 +++++++++---------- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e76d946866b..e20df483030d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) - [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) +- Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index 713de8cdd0fda..455e7301a490d 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -299,33 +299,37 @@ public static boolean assertNotMasterUpdateThread(String reason) { } private void runTasks(TaskInputs taskInputs) { - final String longSummary = logger.isTraceEnabled() ? taskInputs.taskSummaryGenerator.apply(true) : ""; - final String shortSummary = taskInputs.taskSummaryGenerator.apply(false); + final String summary; + if (logger.isTraceEnabled()) { + summary = taskInputs.taskSummaryGenerator.apply(true); + } else { + summary = taskInputs.taskSummaryGenerator.apply(false); + } if (!lifecycle.started()) { - logger.debug("processing [{}]: ignoring, cluster-manager service not started", shortSummary); + logger.debug("processing [{}]: ignoring, cluster-manager service not started", summary); return; } if (logger.isTraceEnabled()) { - logger.trace("executing cluster state update for [{}]", longSummary); + logger.trace("executing cluster state update for [{}]", summary); } else { - logger.debug("executing cluster state update for [{}]", shortSummary); + logger.debug("executing cluster state update for [{}]", summary); } final ClusterState previousClusterState = state(); if (!previousClusterState.nodes().isLocalNodeElectedClusterManager() && taskInputs.runOnlyWhenClusterManager()) { - logger.debug("failing [{}]: local node is no longer cluster-manager", shortSummary); + logger.debug("failing [{}]: local node is no longer cluster-manager", summary); taskInputs.onNoLongerClusterManager(); return; } final long computationStartTime = threadPool.preciseRelativeTimeInNanos(); - final TaskOutputs taskOutputs = calculateTaskOutputs(taskInputs, previousClusterState, shortSummary); + final TaskOutputs taskOutputs = calculateTaskOutputs(taskInputs, previousClusterState, summary); taskOutputs.notifyFailedTasks(); final TimeValue computationTime = getTimeSince(computationStartTime); - logExecutionTime(computationTime, "compute cluster state update", shortSummary); + logExecutionTime(computationTime, "compute cluster state update", summary); clusterManagerMetrics.recordLatency( clusterManagerMetrics.clusterStateComputeHistogram, @@ -337,17 +341,17 @@ private void runTasks(TaskInputs taskInputs) { final long notificationStartTime = threadPool.preciseRelativeTimeInNanos(); taskOutputs.notifySuccessfulTasksOnUnchangedClusterState(); final TimeValue executionTime = getTimeSince(notificationStartTime); - logExecutionTime(executionTime, "notify listeners on unchanged cluster state", shortSummary); + logExecutionTime(executionTime, "notify listeners on unchanged cluster state", summary); } else { final ClusterState newClusterState = taskOutputs.newClusterState; if (logger.isTraceEnabled()) { - logger.trace("cluster state updated, source [{}]\n{}", longSummary, newClusterState); + logger.trace("cluster state updated, source [{}]\n{}", summary, newClusterState); } else { - logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), shortSummary); + logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), summary); } final long publicationStartTime = threadPool.preciseRelativeTimeInNanos(); try { - ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(shortSummary, newClusterState, previousClusterState); + ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(summary, newClusterState, previousClusterState); // new cluster state, notify all listeners final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta(); if (nodesDelta.hasChanges() && logger.isInfoEnabled()) { @@ -355,7 +359,7 @@ private void runTasks(TaskInputs taskInputs) { if (nodesDeltaSummary.length() > 0) { logger.info( "{}, term: {}, version: {}, delta: {}", - shortSummary, + summary, newClusterState.term(), newClusterState.version(), nodesDeltaSummary @@ -366,7 +370,7 @@ private void runTasks(TaskInputs taskInputs) { logger.debug("publishing cluster state version [{}]", newClusterState.version()); publish(clusterChangedEvent, taskOutputs, publicationStartTime); } catch (Exception e) { - handleException(shortSummary, publicationStartTime, newClusterState, e); + handleException(summary, publicationStartTime, newClusterState, e); } } } diff --git a/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java b/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java index 3513bfffb7157..ac54693b8ad1e 100644 --- a/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java +++ b/server/src/main/java/org/opensearch/cluster/service/TaskBatcher.java @@ -195,16 +195,12 @@ void runIfNotProcessed(BatchedTask updateTask) { if (toExecute.isEmpty() == false) { Function taskSummaryGenerator = (longSummaryRequired) -> { if (longSummaryRequired == null || !longSummaryRequired) { - return buildShortSummary(updateTask.batchingKey, toExecute.size()); + final List sampleTasks = toExecute.stream() + .limit(Math.min(1000, toExecute.size())) + .collect(Collectors.toList()); + return buildShortSummary(updateTask.batchingKey, toExecute.size(), getSummary(updateTask, sampleTasks)); } - final Map> processTasksBySource = new HashMap<>(); - for (final BatchedTask task : toExecute) { - processTasksBySource.computeIfAbsent(task.source, s -> new ArrayList<>()).add(task); - } - return processTasksBySource.entrySet().stream().map(entry -> { - String tasks = updateTask.describeTasks(entry.getValue()); - return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]"; - }).reduce((s1, s2) -> s1 + ", " + s2).orElse(""); + return getSummary(updateTask, toExecute); }; taskBatcherListener.onBeginProcessing(toExecute); run(updateTask.batchingKey, toExecute, taskSummaryGenerator); @@ -212,8 +208,24 @@ void runIfNotProcessed(BatchedTask updateTask) { } } - private String buildShortSummary(final Object batchingKey, final int taskCount) { - return "Tasks batched with key: " + batchingKey.toString().split("\\$")[0] + " and count: " + taskCount; + private String getSummary(final BatchedTask updateTask, final List toExecute) { + final Map> processTasksBySource = new HashMap<>(); + for (final BatchedTask task : toExecute) { + processTasksBySource.computeIfAbsent(task.source, s -> new ArrayList<>()).add(task); + } + return processTasksBySource.entrySet().stream().map(entry -> { + String tasks = updateTask.describeTasks(entry.getValue()); + return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]"; + }).reduce((s1, s2) -> s1 + ", " + s2).orElse(""); + } + + private String buildShortSummary(final Object batchingKey, final int taskCount, final String sampleTasks) { + return "Tasks batched with key: " + + batchingKey.toString().split("\\$")[0] + + ", count:" + + taskCount + + " and sample tasks: " + + sampleTasks; } /** diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index db9abe0310e40..bb9e34d93431f 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -391,7 +391,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test1 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [1s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [1s] to compute cluster state update for [test1]" ) ); mockAppender.addExpectation( @@ -399,7 +399,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test1 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [0s] to notify listeners on unchanged cluster state for [test1]" ) ); @@ -416,7 +416,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test2 failure", MasterService.class.getCanonicalName(), Level.TRACE, - "failed to execute cluster state update (on version: [*], uuid: [*]) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]*" + "failed to execute cluster state update (on version: [*], uuid: [*]) for [test2]*" ) ); mockAppender.addExpectation( @@ -424,7 +424,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test2 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [2s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [2s] to compute cluster state update for [test2]" ) ); mockAppender.addExpectation( @@ -432,7 +432,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test2 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [0s] to notify listeners on unchanged cluster state for [test2]" ) ); @@ -449,7 +449,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test3 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [3s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [3s] to compute cluster state update for [test3]" ) ); mockAppender.addExpectation( @@ -457,7 +457,7 @@ public void testClusterStateUpdateLoggingWithTraceEnabled() throws Exception { "test3 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [4s] to notify listeners on successful publication of cluster state (version: *, uuid: *) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [4s] to notify listeners on successful publication of cluster state (version: *, uuid: *) for [test3]" ) ); @@ -548,7 +548,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test1 start", MasterService.class.getCanonicalName(), Level.DEBUG, - "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test1]" ) ); mockAppender.addExpectation( @@ -556,7 +556,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test1 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [1s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [1s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test1]" ) ); mockAppender.addExpectation( @@ -564,7 +564,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test1 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test1]" ) ); @@ -573,7 +573,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test2 start", MasterService.class.getCanonicalName(), Level.DEBUG, - "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test2]" ) ); mockAppender.addExpectation( @@ -581,7 +581,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test2 failure", MasterService.class.getCanonicalName(), Level.DEBUG, - "failed to execute cluster state update (on version: [*], uuid: [*]) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]*" + "failed to execute cluster state update (on version: [*], uuid: [*]) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test2]*" ) ); mockAppender.addExpectation( @@ -589,7 +589,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test2 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [2s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [2s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test2]" ) ); mockAppender.addExpectation( @@ -597,7 +597,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test2 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [0s] to notify listeners on unchanged cluster state for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test2]" ) ); @@ -606,7 +606,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test3 start", MasterService.class.getCanonicalName(), Level.DEBUG, - "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test3]" ) ); mockAppender.addExpectation( @@ -614,7 +614,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test3 computation", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [3s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [3s] to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test3]" ) ); mockAppender.addExpectation( @@ -622,7 +622,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test3 notification", MasterService.class.getCanonicalName(), Level.DEBUG, - "took [4s] to notify listeners on successful publication of cluster state (version: *, uuid: *) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "took [4s] to notify listeners on successful publication of cluster state (version: *, uuid: *) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test3]" ) ); @@ -631,7 +631,7 @@ public void testClusterStateUpdateLoggingWithDebugEnabled() throws Exception { "test4", MasterService.class.getCanonicalName(), Level.DEBUG, - "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "executing cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test4]" ) ); @@ -1238,7 +1238,7 @@ public void testLongClusterStateUpdateLogging() throws Exception { "test2", MasterService.class.getCanonicalName(), Level.WARN, - "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test2]" ) ); mockAppender.addExpectation( @@ -1246,7 +1246,7 @@ public void testLongClusterStateUpdateLogging() throws Exception { "test3", MasterService.class.getCanonicalName(), Level.WARN, - "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test3]" ) ); mockAppender.addExpectation( @@ -1254,7 +1254,7 @@ public void testLongClusterStateUpdateLogging() throws Exception { "test4", MasterService.class.getCanonicalName(), Level.WARN, - "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]" + "*took [*], which is over [10s], to compute cluster state update for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test4]" ) ); mockAppender.addExpectation( @@ -1432,7 +1432,7 @@ public void testLongClusterStateUpdateLoggingForFailedPublication() throws Excep "test1 should log due to slow and failing publication", MasterService.class.getCanonicalName(), Level.WARN, - "took [*] and then failed to publish updated cluster state (version: *, uuid: *) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests and count: 1]:*" + "took [*] and then failed to publish updated cluster state (version: *, uuid: *) for [Tasks batched with key: org.opensearch.cluster.service.MasterServiceTests, count:1 and sample tasks: test1]:*" ) ); From 72559bf60c84a59908a0685db01d2ed5b1419111 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:46:04 +0530 Subject: [PATCH 291/357] create publication repos during join task execution (#16383) * create publication repos during join task Signed-off-by: Rajiv Kumar Vaidyanathan --- .../RemotePublicationConfigurationIT.java | 10 +- .../coordination/JoinTaskExecutor.java | 42 +++- .../remotestore/RemoteStoreNodeService.java | 2 +- .../coordination/JoinTaskExecutorTests.java | 233 +++++++++++++++++- 4 files changed, 272 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java index 98859f517aad4..57bf9eccbf5b4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -14,6 +14,7 @@ import org.opensearch.remotemigration.MigrationBaseTestCase; import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.repositories.fs.ReloadableFsRepository; import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -97,23 +98,26 @@ public Settings.Builder remotePublishConfiguredNodeSetting() { .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME) - .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(routingTableRepoTypeAttributeKey, FsRepository.TYPE) .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); return builder; } public Settings.Builder remoteWithRoutingTableNodeSetting() { // Remote Cluster with Routing table + return Settings.builder() .put( - buildRemoteStoreNodeAttributes( + remoteStoreClusterSettings( REPOSITORY_NAME, segmentRepoPath, + ReloadableFsRepository.TYPE, REPOSITORY_2_NAME, translogRepoPath, + ReloadableFsRepository.TYPE, REPOSITORY_NAME, segmentRepoPath, - false + ReloadableFsRepository.TYPE ) ) .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 13033b670d44b..d597b51c32ccd 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -42,6 +42,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RerouteService; @@ -57,6 +58,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -185,11 +187,30 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // for every set of node join task which we can optimize to not compute if cluster state already has // repository information. Optional remoteDN = currentNodes.getNodes().values().stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst(); - DiscoveryNode dn = remoteDN.orElseGet(() -> (currentNodes.getNodes().values()).stream().findFirst().get()); - RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( - dn, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) - ); + Optional remotePublicationDN = currentNodes.getNodes() + .values() + .stream() + .filter(DiscoveryNode::isRemoteStatePublicationEnabled) + .findFirst(); + RepositoriesMetadata existingRepositoriesMetadata = currentState.getMetadata().custom(RepositoriesMetadata.TYPE); + Map repositories = new LinkedHashMap<>(); + if (existingRepositoriesMetadata != null) { + existingRepositoriesMetadata.repositories().forEach(r -> repositories.putIfAbsent(r.name(), r)); + } + if (remoteDN.isPresent()) { + RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + remoteDN.get(), + existingRepositoriesMetadata + ); + repositoriesMetadata.repositories().forEach(r -> repositories.putIfAbsent(r.name(), r)); + } + if (remotePublicationDN.isPresent()) { + RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + remotePublicationDN.get(), + existingRepositoriesMetadata + ); + repositoriesMetadata.repositories().forEach(r -> repositories.putIfAbsent(r.name(), r)); + } assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -219,15 +240,16 @@ public ClusterTasksResult execute(ClusterState currentState, List jo ensureNodeCommissioned(node, currentState.metadata()); nodesBuilder.add(node); - if (remoteDN.isEmpty() && node.isRemoteStoreNode()) { + if ((remoteDN.isEmpty() && node.isRemoteStoreNode()) + || (remotePublicationDN.isEmpty() && node.isRemoteStatePublicationEnabled())) { // This is hit only on cases where we encounter first remote node logger.info("Updating system repository now for remote store"); - repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( node, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + existingRepositoriesMetadata ); + repositoriesMetadata.repositories().forEach(r -> repositories.putIfAbsent(r.name(), r)); } - nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion()); @@ -241,7 +263,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } results.success(joinTask); } - + RepositoriesMetadata repositoriesMetadata = new RepositoriesMetadata(new ArrayList<>(repositories.values())); if (nodesChanged) { rerouteService.reroute( "post-join reroute", diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index cc5d8b0e62e90..c1c041ce01198 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -168,7 +168,7 @@ public void createAndVerifyRepositories(DiscoveryNode localNode) { * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { - if (joiningNode.isRemoteStoreNode()) { + if (joiningNode.isRemoteStoreNode() || joiningNode.isRemoteStatePublicationEnabled()) { List updatedRepositoryMetadataList = new ArrayList<>(); List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() .repositories(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index d192a2556c36b..f6fb203bfe1a9 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -959,6 +959,198 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); } + public void testUpdatesRepoRemoteNodeJoinPublicationCluster() throws Exception { + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreNodeService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remotePublicationNodeAttributes(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .build(); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(clusterManagerNode, "elect leader")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validatePublicationRepositoryMetadata(result.resultingState, clusterManagerNode); + + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + + ClusterState currentState = ClusterState.builder(result.resultingState).metadata(metadata).build(); + + final DiscoveryNode remoteStoreNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterStateTaskExecutor.ClusterTasksResult resultAfterRemoteNodeJoin = joinTaskExecutor.execute( + currentState, + List.of(new JoinTaskExecutor.Task(remoteStoreNode, "test")) + ); + assertThat(resultAfterRemoteNodeJoin.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult1 = resultAfterRemoteNodeJoin.executionResults.values().iterator().next(); + assertTrue(taskResult1.isSuccess()); + validateRepositoriesMetadata(resultAfterRemoteNodeJoin.resultingState, remoteStoreNode, clusterManagerNode); + } + + public void testUpdatesRepoPublicationNodeJoinRemoteCluster() throws Exception { + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreNodeService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .build(); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(clusterManagerNode, "elect leader")) + ); + final Settings settings = Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE) + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + .build(); + final Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + Metadata metadata = Metadata.builder().persistentSettings(settings).build(); + + ClusterState currentState = ClusterState.builder(result.resultingState).metadata(metadata).build(); + + final DiscoveryNode remotePublicationNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remotePublicationNodeAttributes(), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterStateTaskExecutor.ClusterTasksResult resultAfterRemoteNodeJoin = joinTaskExecutor.execute( + currentState, + List.of(new JoinTaskExecutor.Task(remotePublicationNode, "test")) + ); + assertThat(resultAfterRemoteNodeJoin.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult1 = resultAfterRemoteNodeJoin.executionResults.values().iterator().next(); + assertTrue(taskResult1.isSuccess()); + validateRepositoriesMetadata(resultAfterRemoteNodeJoin.resultingState, clusterManagerNode, remotePublicationNode); + } + + public void testUpdatesClusterStateWithMultiplePublicationNodeJoin() throws Exception { + Map remoteStoreNodeAttributes = remotePublicationNodeAttributes(); + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreNodeService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + List repositoriesMetadata = new ArrayList<>(); + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .metadata(Metadata.builder().putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata))) + .build(); + + final DiscoveryNode joiningNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(joiningNode, "test")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validatePublicationRepositoryMetadata(result.resultingState, clusterManagerNode); + } + public void testNodeJoinInMixedMode() { List versions = allOpenSearchVersions(); assert versions.size() >= 2 : "test requires at least two open search versions"; @@ -1191,7 +1383,9 @@ private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode final RepositoriesMetadata repositoriesMetadata = updatedState.metadata().custom(RepositoriesMetadata.TYPE); assertTrue(repositoriesMetadata.repositories().size() == expectedRepositories); - if (repositoriesMetadata.repositories().size() == 2 || repositoriesMetadata.repositories().size() == 3) { + if (repositoriesMetadata.repositories().size() == 2 + || repositoriesMetadata.repositories().size() == 3 + || repositoriesMetadata.repositories().size() == 4) { final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(existingNode, SEGMENT_REPO); final RepositoryMetadata translogRepositoryMetadata = buildRepositoryMetadata(existingNode, TRANSLOG_REPO); for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { @@ -1212,6 +1406,43 @@ private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode } } + private void validatePublicationRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode) throws Exception { + final RepositoriesMetadata repositoriesMetadata = updatedState.metadata().custom(RepositoriesMetadata.TYPE); + assertTrue(repositoriesMetadata.repositories().size() == 2); + final RepositoryMetadata clusterStateRepoMetadata = buildRepositoryMetadata(existingNode, CLUSTER_STATE_REPO); + final RepositoryMetadata routingTableRepoMetadata = buildRepositoryMetadata(existingNode, ROUTING_TABLE_REPO); + for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { + if (repositoryMetadata.name().equals(clusterStateRepoMetadata.name())) { + assertTrue(clusterStateRepoMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoryMetadata.name().equals(routingTableRepoMetadata.name())) { + assertTrue(routingTableRepoMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } + } + } + + private void validateRepositoriesMetadata(ClusterState updatedState, DiscoveryNode remoteNode, DiscoveryNode publicationNode) + throws Exception { + + final RepositoriesMetadata repositoriesMetadata = updatedState.metadata().custom(RepositoriesMetadata.TYPE); + assertEquals(4, repositoriesMetadata.repositories().size()); + final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(remoteNode, SEGMENT_REPO); + final RepositoryMetadata translogRepositoryMetadata = buildRepositoryMetadata(remoteNode, TRANSLOG_REPO); + final RepositoryMetadata clusterStateRepositoryMetadata = buildRepositoryMetadata(remoteNode, CLUSTER_STATE_REPO); + final RepositoryMetadata routingTableRepositoryMetadata = buildRepositoryMetadata(publicationNode, ROUTING_TABLE_REPO); + + for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { + if (repositoryMetadata.name().equals(segmentRepositoryMetadata.name())) { + assertTrue(segmentRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoryMetadata.name().equals(translogRepositoryMetadata.name())) { + assertTrue(translogRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoryMetadata.name().equals(clusterStateRepositoryMetadata.name())) { + assertTrue(clusterStateRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoryMetadata.name().equals(routingTableRepositoryMetadata.name())) { + assertTrue(repositoryMetadata.equalsIgnoreGenerations(routingTableRepositoryMetadata)); + } + } + } + private DiscoveryNode newDiscoveryNode(Map attributes) { return new DiscoveryNode( randomAlphaOfLength(10), From 0fcb3ab82e0e2373f79287dd035c3b5e2a5fa306 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:15:44 -0400 Subject: [PATCH 292/357] Bump ch.qos.logback:logback-core from 1.5.10 to 1.5.12 in /test/fixtures/hdfs-fixture (#16503) * Bump ch.qos.logback:logback-core in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.5.10 to 1.5.12. - [Commits](https://github.com/qos-ch/logback/compare/v_1.5.10...v_1.5.12) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e20df483030d6..c1e4f459d1422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.2 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945), [#16406](https://github.com/opensearch-project/OpenSearch/pull/16406)) -- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.12 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307), [#16503](https://github.com/opensearch-project/OpenSearch/pull/16503)) - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) - Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 2bd0268ca136b..d27273f357758 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -74,7 +74,7 @@ dependencies { api 'org.apache.zookeeper:zookeeper:3.9.2' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" - api "ch.qos.logback:logback-core:1.5.10" + api "ch.qos.logback:logback-core:1.5.12" api "ch.qos.logback:logback-classic:1.2.13" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.27.1' From 9f7d3b6dcebf23a9ed40fa34a80f187569f40048 Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:35:48 +0530 Subject: [PATCH 293/357] using the routing allocation to cancel existing recoveries (#16468) Signed-off-by: Rajiv Kumar Vaidyanathan --- .../coordination/RareClusterStateIT.java | 179 ++++++++++++++++++ .../gateway/AsyncShardBatchFetch.java | 12 +- .../gateway/ReplicaShardBatchAllocator.java | 31 ++- 3 files changed, 212 insertions(+), 10 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java index b3cb15d028090..cc0264f375103 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java @@ -40,6 +40,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; @@ -48,29 +49,39 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.index.Index; +import org.opensearch.core.transport.TransportResponse; import org.opensearch.discovery.Discovery; import org.opensearch.index.IndexService; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.IndicesService; +import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.disruption.BlockClusterStateProcessing; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.transport.TransportService; import org.opensearch.transport.TransportSettings; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.opensearch.action.DocWriteResponse.Result.CREATED; +import static org.opensearch.cluster.action.shard.ShardStateAction.SHARD_STARTED_ACTION_NAME; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; @@ -409,4 +420,172 @@ public void testDelayedMappingPropagationOnReplica() throws Exception { assertThat(dynamicMappingsFut.get(10, TimeUnit.SECONDS).getResult(), equalTo(CREATED)); } + public void testDisassociateNodesWhileShardInit() throws InterruptedException { + final String clusterManagerName = internalCluster().startClusterManagerOnlyNode( + Settings.builder() + .put(TransportSettings.CONNECT_TIMEOUT.getKey(), "1s") + .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true) + .build() + ); + internalCluster().startDataOnlyNode(Settings.builder().put(TransportSettings.CONNECT_TIMEOUT.getKey(), "1s").build()); + internalCluster().startDataOnlyNode(Settings.builder().put(TransportSettings.CONNECT_TIMEOUT.getKey(), "1s").build()); + String node2 = internalCluster().startDataOnlyNode( + Settings.builder().put(TransportSettings.CONNECT_TIMEOUT.getKey(), "1s").build() + ); + + final ClusterService clusterService = internalCluster().clusterService(clusterManagerName); + blockShardStartedResponse(clusterManagerName, clusterService); + + final String index = "index"; + + // create index with 3 primary and 1 replica each + prepareCreate(index).setSettings( + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + // .put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries") + ).get(); + ensureGreen(index); + + // close to have some unassigned started shards shards.. + client().admin().indices().prepareClose(index).get(); + + // block so that replicas are always in init and not started + blockReplicaStart.set(true); + final AllocationService allocationService = internalCluster().getInstance(AllocationService.class, clusterManagerName); + clusterService.submitStateUpdateTask("test-delete-node-and-reroute", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + ClusterState.Builder builder = ClusterState.builder(currentState); + // open index + final IndexMetadata indexMetadata = IndexMetadata.builder(currentState.metadata().index(index)) + .state(IndexMetadata.State.OPEN) + .build(); + + builder.metadata(Metadata.builder(currentState.metadata()).put(indexMetadata, true)); + builder.blocks(ClusterBlocks.builder().blocks(currentState.blocks()).removeIndexBlocks(index)); + ClusterState updatedState = builder.build(); + RoutingTable.Builder routingTable = RoutingTable.builder(updatedState.routingTable()); + routingTable.addAsRecovery(updatedState.metadata().index(index)); + updatedState = ClusterState.builder(updatedState).routingTable(routingTable.build()).build(); + ClusterState state = allocationService.reroute(updatedState, "reroute"); + return state; + } + + @Override + public void onFailure(String source, Exception e) { + logger.error(e.getMessage(), e); + } + }); + + ensureYellow(index); + assertTrue(waitUntil(() -> { + ClusterState state = clusterService.state(); + return state.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size() == 3; + + })); + + logger.info("Initializing shards"); + logger.info(clusterService.state().getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING)); + + // trigger 2nd reroute after shard in initialized + clusterService.submitStateUpdateTask("test-delete-node-and-reroute", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return allocationService.reroute(currentState, "reroute"); + } + + @Override + public void onFailure(String source, Exception e) {} + }); + + ensureYellow(index); + assertTrue(waitUntil(() -> clusterService.state().getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size() == 3)); + clusterService.submitStateUpdateTask("test-remove-injected-node", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + // remove the primary node of replica shard which is in init + ShardRouting next = currentState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0); + ShardRouting primaryShard = currentState.getRoutingNodes().activePrimary(next.shardId()); + + ClusterState.Builder builder = ClusterState.builder(currentState); + builder.nodes(DiscoveryNodes.builder(currentState.nodes()).remove(primaryShard.currentNodeId())); + currentState = builder.build(); + logger.info("removed the node {}", primaryShard.currentNodeId()); + logger.info("shard {}", next); + ClusterState state = allocationService.disassociateDeadNodes(currentState, true, "reroute"); + return state; + } + + @Override + public void onFailure(String source, Exception e) {} + }); + assertTrue(waitUntil(() -> { + ClusterState state = clusterService.state(); + logger.info("current state {} ", state); + return clusterService.state().nodes().getSize() == 3; + + })); + + logger.info(clusterService.state().getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING)); + blockReplicaStart.set(false); + + clusterService.submitStateUpdateTask("test-inject-node-and-reroute", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + ClusterState.Builder builder = ClusterState.builder(currentState); + final IndexMetadata indexMetadata = IndexMetadata.builder(currentState.metadata().index(index)) + .state(IndexMetadata.State.OPEN) + .build(); + builder.metadata(Metadata.builder(currentState.metadata()).put(indexMetadata, true)); + builder.blocks(ClusterBlocks.builder().blocks(currentState.blocks()).removeIndexBlocks(index)); + ClusterState updatedState = builder.build(); + RoutingTable.Builder routingTable = RoutingTable.builder(updatedState.routingTable()); + routingTable.addAsRecovery(updatedState.metadata().index(index)); + updatedState = ClusterState.builder(updatedState).routingTable(routingTable.build()).build(); + + return allocationService.reroute(updatedState, "reroute"); + } + + @Override + public void onFailure(String source, Exception e) {} + }); + + ensureGreen(index); + } + + AtomicBoolean blockReplicaStart = new AtomicBoolean(false); + + private void blockShardStartedResponse(String master, ClusterService service) { + MockTransportService primaryService = (MockTransportService) internalCluster().getInstance(TransportService.class, master); + primaryService.addRequestHandlingBehavior(SHARD_STARTED_ACTION_NAME, (handler, request, channel, task) -> { + + if (blockReplicaStart.get()) { + ShardStateAction.StartedShardEntry req = (ShardStateAction.StartedShardEntry) request; + String stringRep = req.toString(); + logger.info("ShardStateAction.StartedShardEntry {}", stringRep); + + String incomingRequest = req.toString(); + Optional matchReplica = service.state() + .routingTable() + .allShardsSatisfyingPredicate(r -> !r.primary()) + .getShardRoutings() + .stream() + .filter(r -> r.allocationId() != null) + .filter(r -> incomingRequest.contains(r.allocationId().getId())) + .findAny(); + + if (matchReplica.isPresent()) { + channel.sendResponse(TransportResponse.Empty.INSTANCE); + } else { + handler.messageReceived(request, channel, task); + } + } else { + handler.messageReceived(request, channel, task); + } + }); + } + + @Override + protected Collection> nodePlugins() { + return List.of(MockTransportService.TestPlugin.class); + } } diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java index d86d41bb1a359..095730bd84f8d 100644 --- a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java +++ b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java @@ -241,11 +241,13 @@ private void fillShardData(Map shardDataFromNode, Map shardData : shardDataFromNode.entrySet()) { if (shardData.getValue() != null) { ShardId shardId = shardData.getKey(); - if (emptyShardResponsePredicate.test(shardData.getValue())) { - this.emptyShardResponse[shardIdKey.get(shardId)] = true; - this.shardData[shardIdKey.get(shardId)] = null; - } else { - this.shardData[shardIdKey.get(shardId)] = shardData.getValue(); + if (shardIdKey.get(shardId) != null) {// the response might be for shard which is no longer present in cache + if (emptyShardResponsePredicate.test(shardData.getValue())) { + this.emptyShardResponse[shardIdKey.get(shardId)] = true; + this.shardData[shardIdKey.get(shardId)] = null; + } else { + this.shardData[shardIdKey.get(shardId)] = shardData.getValue(); + } } } } diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java index 020a543ac5fc5..2b6fdfa8dd411 100644 --- a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RoutingNodes; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision; import org.opensearch.cluster.routing.allocation.NodeAllocationResult; @@ -51,6 +52,12 @@ public abstract class ReplicaShardBatchAllocator extends ReplicaShardAllocator { */ public void processExistingRecoveries(RoutingAllocation allocation, List> shardBatches) { List shardCancellationActions = new ArrayList<>(); + Map> initReplicasFromRouting = new HashMap<>(); + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).stream().filter(r -> !r.primary()).forEach(r -> { + initReplicasFromRouting.putIfAbsent(r.shardId(), new ArrayList<>()); + initReplicasFromRouting.get(r.shardId()).add(r); + }); + // iterate through the batches, each batch needs to be processed together as fetch call should be made for shards from same batch for (List shardBatch : shardBatches) { List eligibleShards = new ArrayList<>(); @@ -58,6 +65,12 @@ public void processExistingRecoveries(RoutingAllocation allocation, List nodeShardStores = convertToNodeStoreFilesMetadataMap(shard, shardState); - - Runnable cancellationAction = cancelExistingRecoveryForBetterMatch(shard, allocation, nodeShardStores); - if (cancellationAction != null) { - shardCancellationActions.add(cancellationAction); + for (ShardRouting initShardsFromAllocation : initReplicasFromRouting.get(shard.shardId())) { + Map nodeShardStores = convertToNodeStoreFilesMetadataMap( + initShardsFromAllocation, + shardState + ); + Runnable cancellationAction = cancelExistingRecoveryForBetterMatch( + initShardsFromAllocation, + allocation, + nodeShardStores + ); + if (cancellationAction != null) { + shardCancellationActions.add(cancellationAction); + } } } } From af7a067e04583d2870af684d70612015e883b63c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 28 Oct 2024 13:44:30 -0400 Subject: [PATCH 294/357] Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs (#16284) * WIP on lib toml Signed-off-by: Craig Perkins * SpotlessApply Signed-off-by: Craig Perkins * Remove unnecessary lines Signed-off-by: Craig Perkins * Specify time when dependabot runs Signed-off-by: Craig Perkins * Refer to version from libs.versions.toml Signed-off-by: Craig Perkins * Use version Signed-off-by: Craig Perkins * Specify version catalog Signed-off-by: Craig Perkins * Call .get() Signed-off-by: Craig Perkins * Define version catalog Signed-off-by: Craig Perkins * Use libraries Signed-off-by: Craig Perkins * Downgrade purposefully Signed-off-by: Craig Perkins * Add mavenCentral Signed-off-by: Craig Perkins * Try w/o libraries section Signed-off-by: Craig Perkins * reinstate Signed-off-by: Craig Perkins * Remove version.properties Signed-off-by: Craig Perkins * Update syntax Signed-off-by: Craig Perkins * Change back to weekly Signed-off-by: Craig Perkins * Add grpc Signed-off-by: Craig Perkins * Get relative to project root. Relative path not working on windows bc windows had gradle wrapper path Signed-off-by: Craig Perkins * Add minimal version.properties with only opensearch version to accommodate external references Signed-off-by: Craig Perkins * singularize version.properties Signed-off-by: Craig Perkins * Get rootDir Signed-off-by: Craig Perkins * Fix issue loading snapshot Signed-off-by: Craig Perkins * Limit logic to generating version.properties file within buildSrc Signed-off-by: Craig Perkins * Remove unused exports Signed-off-by: Craig Perkins * Add import Signed-off-by: Craig Perkins * Remove unused code Signed-off-by: Craig Perkins * Remove mavenCentral from publication section Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Update reactor-netty version Signed-off-by: Craig Perkins * Only keep versions section in toml Signed-off-by: Craig Perkins * Replaces versions catalog TOML parsing with Gradle's VersionCatalogsExtension Signed-off-by: Andriy Redko * Update bundled_jdk Signed-off-by: Craig Perkins * Update bytebuddy and mockito Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins Signed-off-by: Andriy Redko Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Andriy Redko Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + buildSrc/build.gradle | 20 ++++++---- buildSrc/settings.gradle | 8 ++++ buildSrc/version.properties | 79 +----------------------------------- gradle/libs.versions.toml | 80 +++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 86 deletions(-) create mode 100644 gradle/libs.versions.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e4f459d1422..d12a47fb5a006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) +- Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index aaa1d8bdef85c..c62f20e106e8c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -57,7 +57,7 @@ if (project == rootProject) { // we update the version property to reflect if we are building a snapshot or a release build // we write this back out below to load it in the Build.java which will be shown in rest main action // to indicate this being a snapshot build or a release build. -Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties')) +Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project) version = props.getProperty("opensearch") def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { @@ -287,15 +287,19 @@ if (project != rootProject) { } } -// Define this here because we need it early. +// Define this here because we need it early. It uses VersionCatalogsExtension to extract all versions +// and converts to a Java Properties object class VersionPropertiesLoader { - static Properties loadBuildSrcVersion(File input) throws IOException { + static Properties loadBuildSrcVersion(Project project) throws IOException { Properties props = new Properties(); - InputStream is = new FileInputStream(input) - try { - props.load(is) - } finally { - is.close() + + var catalogs = project.extensions.getByType(VersionCatalogsExtension) + var libs = catalogs.named("libs") + libs.getVersionAliases().forEach { + libs.findVersion(it).ifPresent { v -> + // Gradle replaces '_' with '.' so 'google_http_client' becomes 'google.http.client' instead + props.setProperty(it.replaceAll("[.]", "_"), v.requiredVersion) + } } loadBuildSrcVersion(props, System.getProperties()) return props diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle index 963177afaa2de..661137fbadae2 100644 --- a/buildSrc/settings.gradle +++ b/buildSrc/settings.gradle @@ -10,3 +10,11 @@ */ include 'reaper' + +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../gradle/libs.versions.toml")) + } + } +} diff --git a/buildSrc/version.properties b/buildSrc/version.properties index f9a8bee5783b1..61f4fbbf10b1d 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,79 +1,2 @@ +# Please use ../gradle/libs.versions.toml for dependency management opensearch = 3.0.0 -lucene = 9.12.0 - -bundled_jdk_vendor = adoptium -bundled_jdk = 23.0.1+11 - -# optional dependencies -spatial4j = 0.7 -jts = 1.15.0 -jackson = 2.17.2 -jackson_databind = 2.17.2 -snakeyaml = 2.1 -icu4j = 75.1 -supercsv = 2.4.0 -log4j = 2.21.0 -slf4j = 1.7.36 -asm = 9.7 -jettison = 1.5.4 -woodstox = 6.4.0 -kotlin = 1.7.10 -antlr4 = 4.13.1 -guava = 32.1.1-jre -protobuf = 3.25.5 -jakarta_annotation = 1.3.5 -google_http_client = 1.44.1 -tdigest = 3.3 -hdrhistogram = 2.2.2 -grpc = 1.68.0 - -# when updating the JNA version, also update the version in buildSrc/build.gradle -jna = 5.13.0 - -netty = 4.1.114.Final -joda = 2.12.7 - -# project reactor -reactor_netty = 1.1.23 -reactor = 3.5.20 - -# client dependencies -httpclient5 = 5.3.1 -httpcore5 = 5.2.5 -httpclient = 4.5.14 -httpcore = 4.4.16 -httpasyncclient = 4.1.5 -commonslogging = 1.2 -commonscodec = 1.16.1 -commonslang = 3.14.0 -commonscompress = 1.26.1 -commonsio = 2.16.0 -# plugin dependencies -aws = 2.20.86 -reactivestreams = 1.0.4 - -# when updating this version, you need to ensure compatibility with: -# - plugins/ingest-attachment (transitive dependency, check the upstream POM) -# - distribution/tools/plugin-cli -bouncycastle=1.78 -# test dependencies -randomizedrunner = 2.7.1 -junit = 4.13.2 -hamcrest = 2.1 -mockito = 5.14.1 -objenesis = 3.2 -bytebuddy = 1.15.4 - -# benchmark dependencies -jmh = 1.35 - -# compression -zstd = 1.5.5-5 - -jzlib = 1.1.3 - -resteasy = 6.2.4.Final - -# opentelemetry dependencies -opentelemetry = 1.41.0 -opentelemetrysemconv = 1.27.0-alpha diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000000000..8ff3f6e45397d --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,80 @@ +[versions] +opensearch = "3.0.0" +lucene = "9.12.0" + +bundled_jdk_vendor = "adoptium" +bundled_jdk = "23.0.1+11" + +# optional dependencies +spatial4j = "0.7" +jts = "1.15.0" +jackson = "2.17.2" +jackson_databind = "2.17.2" +snakeyaml = "2.1" +icu4j = "75.1" +supercsv = "2.4.0" +log4j = "2.21.0" +slf4j = "1.7.36" +asm = "9.7" +jettison = "1.5.4" +woodstox = "6.4.0" +kotlin = "1.7.10" +antlr4 = "4.13.1" +guava = "32.1.1-jre" +protobuf = "3.25.5" +jakarta_annotation = "1.3.5" +google_http_client = "1.44.1" +tdigest = "3.3" +hdrhistogram = "2.2.2" +grpc = "1.68.0" + +# when updating the JNA version, also update the version in buildSrc/build.gradle +jna = "5.13.0" + +netty = "4.1.114.Final" +joda = "2.12.7" + +# project reactor +reactor_netty = "1.1.23" +reactor = "3.5.20" + +# client dependencies +httpclient5 = "5.3.1" +httpcore5 = "5.2.5" +httpclient = "4.5.14" +httpcore = "4.4.16" +httpasyncclient = "4.1.5" +commonslogging = "1.2" +commonscodec = "1.16.1" +commonslang = "3.14.0" +commonscompress = "1.26.1" +commonsio = "2.16.0" +# plugin dependencies +aws = "2.20.86" +reactivestreams = "1.0.4" + +# when updating this version, you need to ensure compatibility with: +# - plugins/ingest-attachment (transitive dependency, check the upstream POM) +# - distribution/tools/plugin-cli +bouncycastle="1.78" +# test dependencies +randomizedrunner = "2.7.1" +junit = "4.13.2" +hamcrest = "2.1" +mockito = "5.14.1" +objenesis = "3.2" +bytebuddy = "1.15.4" + +# benchmark dependencies +jmh = "1.35" + +# compression +zstd = "1.5.5-5" + +jzlib = "1.1.3" + +resteasy = "6.2.4.Final" + +# opentelemetry dependencies +opentelemetry = "1.41.0" +opentelemetrysemconv = "1.27.0-alpha" From 6385ad37818391738d365e66c9f1ebdcb6dc282b Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 28 Oct 2024 14:45:41 -0500 Subject: [PATCH 295/357] Add 2.18 release notes (#16511) Signed-off-by: Andrew Ross --- CHANGELOG.md | 82 +---------------- .../opensearch.release-notes-2.18.0.md | 92 +++++++++++++++++++ 2 files changed, 93 insertions(+), 81 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.18.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d12a47fb5a006..dc1fc20b6aa95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,105 +5,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) -- [Workload Management] Add Integration Tests for Workload Management CRUD APIs ([#15955](https://github.com/opensearch-project/OpenSearch/pull/15955)) -- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) -- [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) -- [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777)) -- Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) -- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) -- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110)) -- [Tiered Caching] Segmented cache changes ([#16047](https://github.com/opensearch-project/OpenSearch/pull/16047)) -- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) -- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) -- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) -- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) -- Add support to dynamically resize threadpools size. ([#16236](https://github.com/opensearch-project/OpenSearch/pull/16236)) -- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) -- Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) -- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) -- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) -- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) -- Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292)) -- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)) -- URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) -- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) ### Dependencies -- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) -- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) -- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134)) -- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) -- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) -- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.2 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945), [#16406](https://github.com/opensearch-project/OpenSearch/pull/16406)) -- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.12 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307), [#16503](https://github.com/opensearch-project/OpenSearch/pull/16503)) -- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) -- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) -- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) -- Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) -- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) -- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) -- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) -- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333)) -- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.5 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133), [#16311](https://github.com/opensearch-project/OpenSearch/pull/16311)) -- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) -- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) -- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) -- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) -- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) -- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) -- Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) -- Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306)) -- Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310)) -- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#16308](https://github.com/opensearch-project/OpenSearch/pull/16308)) ### Changed -- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) -- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) -- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) -- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) -- Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290)) -- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) -- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) -- Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215)) -- Make multiple settings dynamic for tuning on larger clusters([#16347](https://github.com/opensearch-project/OpenSearch/pull/16347)) ### Deprecated ### Removed ### Fixed -- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) -- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) -- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) -- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) -- Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) -- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) -- Fix update settings with null replica not honoring cluster setting bug ([#14948](https://github.com/opensearch-project/OpenSearch/pull/14948)) -- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) -- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) -- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) -- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) - Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294)) -- Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) -- [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) -- Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) -- [Workload Management] Make query groups persistent across process restarts ([#16370](https://github.com/opensearch-project/OpenSearch/pull/16370)) -- [Workload Management] Enhance rejection mechanism in workload management ([#16417](https://github.com/opensearch-project/OpenSearch/pull/16417)) -- Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) -- Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) -- Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) -- Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) -- [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422)) -- Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic ([#16434](https://github.com/opensearch-project/OpenSearch/pull/16434#pullrequestreview-2386999409)) - Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403)) ### Security -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.17...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.18...2.x diff --git a/release-notes/opensearch.release-notes-2.18.0.md b/release-notes/opensearch.release-notes-2.18.0.md new file mode 100644 index 0000000000000..f0dc1d06acdcb --- /dev/null +++ b/release-notes/opensearch.release-notes-2.18.0.md @@ -0,0 +1,92 @@ +## 2024-10-28 Version 2.18.0 Release Notes + +## [2.18.0] +### Added +- [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925)) +- [Offline Nodes] Adds offline-tasks library containing various interfaces to be used for Offline Background Tasks. ([#13574](https://github.com/opensearch-project/OpenSearch/pull/13574)) +- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621)) +- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651)) +- [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777)) +- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916)) +- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110)) +- Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424)) +- [Tiered Caching] Segmented cache changes ([#16047](https://github.com/opensearch-project/OpenSearch/pull/16047)) +- Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) +- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976)) +- Add support to dynamically resize threadpools size. ([#16236](https://github.com/opensearch-project/OpenSearch/pull/16236)) +- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978)) +- [Workload Management] Add Integration Tests for Workload Management CRUD APIs ([#15955](https://github.com/opensearch-project/OpenSearch/pull/15955)) +- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611)) +- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915)) +- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) +- Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https://github.com/opensearch-project/OpenSearch/pull/15986)) +- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383)) +- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387) +- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641)) +- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289)) +- URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) +- Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292)) + +### Dependencies +- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) +- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) +- Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684)) +- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863)) +- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862)) +- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.2 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945), [#16406](https://github.com/opensearch-project/OpenSearch/pull/16406)) +- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307)) +- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011)) +- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040)) +- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038)) +- Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039)) +- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041)) +- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042)) +- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137)) +- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333)) +- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.5 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133), [#16311](https://github.com/opensearch-project/OpenSearch/pull/16311)) +- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182)) +- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216)) +- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217)) +- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) +- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135)) +- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212)) +- Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310)) +- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#16308](https://github.com/opensearch-project/OpenSearch/pull/16308)) +- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213)) +- Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309)) +- Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306)) + +### Changed +- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049)) +- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) +- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024)) +- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154)) +- Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290)) +- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296)) +- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273)) +- Update last seen cluster state in the commit phase ([#16215](https://github.com/opensearch-project/OpenSearch/pull/16215)) +- Make multiple settings dynamic for tuning on larger clusters([#16347](https://github.com/opensearch-project/OpenSearch/pull/16347)) + +### Fixed +- Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737)) +- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882)) +- Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501)) +- Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988)) +- Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985)) +- Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931)) +- Fix update settings with null replica not honoring cluster setting bug ([#14948](https://github.com/opensearch-project/OpenSearch/pull/14948)) +- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521)) +- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158)) +- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194)) +- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254)) +- Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265)) +- [Streaming Indexing] Fix intermittent 'The bulk request must be terminated by a newline [\n]' failures [#16337](https://github.com/opensearch-project/OpenSearch/pull/16337)) +- Fix wrong default value when setting `index.number_of_routing_shards` to null on index creation ([#16331](https://github.com/opensearch-project/OpenSearch/pull/16331)) +- Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386)) +- [Workload Management] Make query groups persistent across process restarts [#16370](https://github.com/opensearch-project/OpenSearch/pull/16370) +- Fix missing fields in task index mapping to ensure proper task result storage ([#16201](https://github.com/opensearch-project/OpenSearch/pull/16201)) +- Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362)) +- Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378)) +- [Workload Management] Enhance rejection mechanism in workload management ([#16417](https://github.com/opensearch-project/OpenSearch/pull/16417)) +- [Workload Management] Fixing Create/Update QueryGroup TransportActions to execute from non-cluster manager nodes ([16422](https://github.com/opensearch-project/OpenSearch/pull/16422)) +- Fix flaky test in `testApproximateRangeWithSizeOverDefault` by adjusting totalHits assertion logic ([#16434](https://github.com/opensearch-project/OpenSearch/pull/16434#pullrequestreview-2386999409)) From 08dc3bb9cb8a4cd01ab32d435e80a41542bbc866 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Mon, 28 Oct 2024 22:20:50 -0500 Subject: [PATCH 296/357] Remove CHANGELOG entry for change that was backported (#16514) This was backported in #15703 so it should no longer be listed in the changelog. Signed-off-by: Andrew Ross --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc1fc20b6aa95..f8b70e46d3d27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added -- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637)) - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) From 1e7c12232a963ad0409da73b0a921f1b727c607c Mon Sep 17 00:00:00 2001 From: Pavan Yekbote Date: Tue, 29 Oct 2024 14:11:11 -0700 Subject: [PATCH 297/357] Ensure index templates are not applied to system indices (#16418) * fix: ensure system indices are processed without templates Signed-off-by: Pavan Yekbote * refactor: overloaded method for creating without templates Signed-off-by: Pavan Yekbote * test: adding test to check call for notemplates on system index Signed-off-by: Pavan Yekbote * refactor: cchange modifier to package private and add entry in changelog Signed-off-by: Pavan Yekbote * test: adding IT test Signed-off-by: Pavan Yekbote * refactor: remove UT and add private modifiers Signed-off-by: Pavan Yekbote * refactor: spotless changes Signed-off-by: Pavan Yekbote --------- Signed-off-by: Pavan Yekbote --- CHANGELOG.md | 1 + .../opensearch/http/SystemIndexRestIT.java | 70 +++++++++++++++++++ .../metadata/MetadataCreateIndexService.java | 21 +++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b70e46d3d27..36c8a34a90b62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294)) - Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403)) +- Ensure index templates are not applied to system indices ([#16418](https://github.com/opensearch-project/OpenSearch/pull/16418)) ### Security diff --git a/qa/smoke-test-http/src/test/java/org/opensearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/test/java/org/opensearch/http/SystemIndexRestIT.java index 9f2d686251947..45564b2a77f91 100644 --- a/qa/smoke-test-http/src/test/java/org/opensearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/opensearch/http/SystemIndexRestIT.java @@ -123,6 +123,76 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception { } } + public void testSystemIndexCreatedWithoutAnyTemplates() throws Exception { + // create template + { + Request templateRequest = new Request("POST", "_component_template/error_mapping_test_template"); + String jsonBody = "{\n" + + " \"template\": {\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"error\" : {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"status\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + templateRequest.setJsonEntity(jsonBody); + Response resp = getRestClient().performRequest(templateRequest); + assertThat(resp.getStatusLine().getStatusCode(), equalTo(200)); + } + + + // apply template to indices + { + Request applyTemplateRequest = new Request("POST", "_index_template/match_all_test_template"); + String jsonBody = "{\n" + + " \"index_patterns\": [\n" + + " \"*system-idx*\"\n" + + " ],\n" + + " \"template\": {\n" + + " \"settings\": {}\n" + + " },\n" + + " \"priority\": 10,\n" + + " \"composed_of\": [\n" + + " \"error_mapping_test_template\"\n" + + " ],\n" + + " \"version\": 1\n" + + "}"; + + applyTemplateRequest.setJsonEntity(jsonBody); + Response resp = getRestClient().performRequest(applyTemplateRequest); + assertThat(resp.getStatusLine().getStatusCode(), equalTo(200)); + } + + // create system index - success + { + Request indexRequest = new Request("PUT", "/" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + String jsonBody = "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"error\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + indexRequest.setJsonEntity(jsonBody); + Response resp = getRestClient().performRequest(indexRequest); + assertThat(resp.getStatusLine().getStatusCode(), equalTo(200)); + } + } + private void assertDeprecationWarningOnAccess(String queryPattern, String warningIndexName) throws IOException { String expectedWarning = "this request accesses system indices: [" + warningIndexName + "], but in a " + "future major version, direct access to system indices will be prevented by default"; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 11df35527eea7..727a08b615050 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -434,6 +434,14 @@ public ClusterState applyCreateIndexRequest( // in which case templates don't apply, so create the index from the source metadata return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata, metadataTransformer); } else { + // The backing index may have a different name or prefix than the data stream name. + final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index(); + + // Do not apply any templates to system indices + if (systemIndices.isSystemIndex(name)) { + return applyCreateIndexRequestWithNoTemplates(currentState, request, silent, metadataTransformer); + } + // Hidden indices apply templates slightly differently (ignoring wildcard '*' // templates), so we need to check to see if the request is creating a hidden index // prior to resolving which templates it matches @@ -441,8 +449,6 @@ public ClusterState applyCreateIndexRequest( ? IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; - // The backing index may have a different name or prefix than the data stream name. - final String name = request.dataStreamName() != null ? request.dataStreamName() : request.index(); // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template( currentState.metadata(), @@ -676,6 +682,17 @@ public void addRemoteStoreCustomMetadata(IndexMetadata.Builder tmpImdBuilder, bo tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData); } + private ClusterState applyCreateIndexRequestWithNoTemplates( + final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final BiConsumer metadataTransformer + ) throws Exception { + // Using applyCreateIndexRequestWithV1Templates with empty list instead of applyCreateIndexRequestWithV2Template + // with null template as applyCreateIndexRequestWithV2Template has assertions when template is null + return applyCreateIndexRequestWithV1Templates(currentState, request, silent, Collections.emptyList(), metadataTransformer); + } + private ClusterState applyCreateIndexRequestWithV1Templates( final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, From 1ec5bcb8c88c7672b83764ac3b1592fbabd61803 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:13:29 +0800 Subject: [PATCH 298/357] Bump com.google.apis:google-api-services-compute from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 in /plugins/discovery-gce (#16502) * Bump com.google.apis:google-api-services-compute Bumps com.google.apis:google-api-services-compute from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0. --- updated-dependencies: - dependency-name: com.google.apis:google-api-services-compute dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/discovery-gce/build.gradle | 2 +- .../google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 | 1 - .../google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c8a34a90b62..2b3d5a7327c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) ### Dependencies +- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502)) ### Changed diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 76beb78bf533c..4e05544a33f1d 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -18,7 +18,7 @@ opensearchplugin { } dependencies { - api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0" + api "com.google.apis:google-api-services-compute:v1-rev20241015-2.0.0" api "com.google.api-client:google-api-client:1.35.2" api "com.google.oauth-client:google-oauth-client:1.36.0" api "com.google.http-client:google-http-client:${versions.google_http_client}" diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 deleted file mode 100644 index 834d718641a51..0000000000000 --- a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -edf93bc92c9b87fee51aa6c3545b565e58075c05 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..1de9a570242fd --- /dev/null +++ b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 @@ -0,0 +1 @@ +83d293916d59ced480e48fd8c0aefb643e27566c \ No newline at end of file From f57b8895340341d79908401d8d27b427feaef2e9 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Wed, 30 Oct 2024 19:49:33 +0800 Subject: [PATCH 299/357] Bump com.azure:azure-storage-common from 12.25.1 to 12.27.1 (#16521) * Bump com.azure:azure-storage-common from 12.25.1 to 12.27.1 Signed-off-by: Gao Binlong * Update changelog Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong Signed-off-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-common-12.25.1.jar.sha1 | 1 - .../licenses/azure-storage-common-12.27.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-common-12.27.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b3d5a7327c06..a16453a369114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) ### Dependencies +- Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 3d6b3264d3f60..3c346a21e5566 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -47,7 +47,7 @@ dependencies { api 'com.azure:azure-core:1.51.0' api 'com.azure:azure-json:1.3.0' api 'com.azure:azure-xml:1.1.0' - api 'com.azure:azure-storage-common:12.25.1' + api 'com.azure:azure-storage-common:12.27.1' api 'com.azure:azure-core-http-netty:1.15.5' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1 deleted file mode 100644 index 822a60d81ca27..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-common-12.25.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -96e2df76ce9a8fa084ae289bb59295d565f2b8d5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.27.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.27.1.jar.sha1 new file mode 100644 index 0000000000000..d7602da1418d1 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-common-12.27.1.jar.sha1 @@ -0,0 +1 @@ +c477c5d8c0f2076da1c5345c1097be6a319fe7c4 \ No newline at end of file From 4b284c542707037b4fcc8163db2ab7b14abe8263 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:50:08 -0400 Subject: [PATCH 300/357] Bump com.azure:azure-storage-blob from 12.23.0 to 12.28.1 in /plugins/repository-azure (#16501) * Bump com.azure:azure-storage-blob in /plugins/repository-azure Bumps [com.azure:azure-storage-blob](https://github.com/Azure/azure-sdk-for-java) from 12.23.0 to 12.28.1. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.23.0...azure-storage-blob_12.28.1) --- updated-dependencies: - dependency-name: com.azure:azure-storage-blob dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-blob-12.23.0.jar.sha1 | 1 - .../licenses/azure-storage-blob-12.28.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.23.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.28.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index a16453a369114..303e2708b6677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502)) +- Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) ### Changed diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 3c346a21e5566..efcd01d2bad5e 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -56,7 +56,7 @@ dependencies { api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') - api 'com.azure:azure-storage-blob:12.23.0' + api 'com.azure:azure-storage-blob:12.28.1' api 'com.azure:azure-identity:1.13.2' // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.23.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.23.0.jar.sha1 deleted file mode 100644 index 5f32d64b00918..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-blob-12.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3eeb49d5109e812343fb436e4bbb2eecac8fe386 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.28.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.28.1.jar.sha1 new file mode 100644 index 0000000000000..95ac42063d36f --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.28.1.jar.sha1 @@ -0,0 +1 @@ +4f1eef206d0a71e4ef6c5e79578dd1b9c0370869 \ No newline at end of file From 80ca32fe9f99165a322e115605027a4df7695f1e Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Thu, 31 Oct 2024 18:32:03 -0700 Subject: [PATCH 301/357] remove resource usages object from headers (#16532) Signed-off-by: Chenyang Ji --- CHANGELOG.md | 1 + .../search/SearchTaskRequestOperationsListener.java | 8 ++++++++ .../opensearch/common/util/concurrent/ThreadContext.java | 9 +++++++++ .../opensearch/tasks/TaskResourceTrackingService.java | 7 +++++++ 4 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303e2708b6677..5209f46229edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294)) - Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403)) - Ensure index templates are not applied to system indices ([#16418](https://github.com/opensearch-project/OpenSearch/pull/16418)) +- Remove resource usages object from search response headers ([#16532](https://github.com/opensearch-project/OpenSearch/pull/16532)) ### Security diff --git a/server/src/main/java/org/opensearch/action/search/SearchTaskRequestOperationsListener.java b/server/src/main/java/org/opensearch/action/search/SearchTaskRequestOperationsListener.java index 4434d71793b23..ee111b563b747 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchTaskRequestOperationsListener.java +++ b/server/src/main/java/org/opensearch/action/search/SearchTaskRequestOperationsListener.java @@ -25,6 +25,14 @@ public SearchTaskRequestOperationsListener(TaskResourceTrackingService taskResou @Override public void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + // Refresh the coordinator node level resource usages taskResourceTrackingService.refreshResourceStats(context.getTask()); + // Remove the shard level resource usages from thread context + taskResourceTrackingService.removeTaskResourceUsage(); + } + + @Override + public void onRequestFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + taskResourceTrackingService.removeTaskResourceUsage(); } } diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java index 070e18481f2a3..75a7ef94978d4 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/ThreadContext.java @@ -547,6 +547,15 @@ public void updateResponseHeader(final String key, final String value, final Fun threadLocal.set(threadLocal.get().putResponse(key, value, uniqueValue, maxWarningHeaderCount, maxWarningHeaderSize, true)); } + /** + * Remove the {@code value} for the specified {@code key}. + * + * @param key the header name + */ + public void removeResponseHeader(final String key) { + threadLocal.get().responseHeaders.remove(key); + } + /** * Saves the current thread context and wraps command in a Runnable that restores that context before running command. If * command has already been passed through this method then it is returned unaltered rather than wrapped twice. diff --git a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java index ca1957cdb1633..a184673a8fa2f 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResourceTrackingService.java @@ -328,6 +328,13 @@ public void writeTaskResourceUsage(SearchShardTask task, String nodeId) { } } + /** + * Remove the current task level resource usages. + */ + public void removeTaskResourceUsage() { + threadPool.getThreadContext().removeResponseHeader(TASK_RESOURCE_USAGE); + } + /** * Get the task resource usages from {@link ThreadContext} * From 0363aa79c434343dc7f5b8df9f77b2d4652f4c7e Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Thu, 31 Oct 2024 22:54:00 -0700 Subject: [PATCH 302/357] Adds cluster setting to allow caching requests with size>0 in request cache (#16484) * Add cluster setting to allow size>0 in request cache Signed-off-by: Peter Alfonsi * Add to changelog Signed-off-by: Peter Alfonsi * addressed dbwiddis's comments Signed-off-by: Peter Alfonsi * make canCacheSizeNonzeroRequests volatile Signed-off-by: Peter Alfonsi * fix changelog merge Signed-off-by: Peter Alfonsi * Changed setting name Signed-off-by: Peter Alfonsi * more renaming Signed-off-by: Peter Alfonsi * fix spotless check Signed-off-by: Peter Alfonsi * rerun gradle check Signed-off-by: Peter Alfonsi --------- Signed-off-by: Peter Alfonsi Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi --- CHANGELOG.md | 1 + .../indices/IndicesRequestCacheIT.java | 18 +++++ .../common/settings/ClusterSettings.java | 1 + .../indices/IndicesRequestCache.java | 12 ++++ .../opensearch/indices/IndicesService.java | 16 ++++- .../indices/IndicesServiceTests.java | 69 +++++++++++++++---- 6 files changed, 101 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5209f46229edf..84fd3810ff158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) +- Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483/files)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 557f9e19ee424..bab085bf265af 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -43,6 +43,7 @@ import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.node.stats.NodeStats; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; @@ -89,6 +90,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; +import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; @@ -579,6 +581,22 @@ public void testCanCache() throws Exception { OpenSearchAssertions.assertAllSuccessful(r4); assertThat(r4.getHits().getTotalHits().value, equalTo(7L)); assertCacheState(client, index, 0, 4); + + // If size > 0 we should cache if this is enabled via cluster setting + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder().put(INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING.getKey(), true) + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + final SearchResponse r7 = client.prepareSearch(index) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setSize(1) + .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-22").lte("2016-03-26")) + .get(); + OpenSearchAssertions.assertAllSuccessful(r7); + assertThat(r7.getHits().getTotalHits().value, equalTo(5L)); + assertCacheState(client, index, 0, 6); } public void testCacheWithFilteredAlias() throws InterruptedException { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index c1f4e52706465..cac4b3914df5a 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -519,6 +519,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesRequestCache.INDICES_CACHE_QUERY_EXPIRE, IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING, IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING, + IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING, HunspellService.HUNSPELL_LAZY_LOAD, HunspellService.HUNSPELL_IGNORE_CASE, HunspellService.HUNSPELL_DICTIONARY_OPTIONS, diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index 156fe32ff5809..4dde4445cd483 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -147,6 +147,18 @@ public final class IndicesRequestCache implements RemovalListener 0 queries. + * If enabled, fundamentally non-cacheable queries like DFS queries, queries using the `now` keyword, and + * scroll requests are still not cached. + */ + public static final Setting INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING = Setting.boolSetting( + "indices.requests.cache.enable_for_all_requests", + false, + Property.NodeScope, + Property.Dynamic + ); + private final static long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Key.class); private final ConcurrentMap registeredClosedListeners = ConcurrentCollections.newConcurrentMap(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 1c12e8ca17194..1a4c9067939a9 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -205,6 +205,7 @@ import static org.opensearch.index.IndexService.IndexCreationContext.CREATE_INDEX; import static org.opensearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -360,6 +361,7 @@ public class IndicesService extends AbstractLifecycleComponent private final FileCache fileCache; private final CompositeIndexSettings compositeIndexSettings; private final Consumer replicator; + private volatile boolean requestCachingEnabledForAllQueries; @Override protected void doStart() { @@ -507,6 +509,9 @@ protected void closeInternal() { this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; this.replicator = replicator; + this.requestCachingEnabledForAllQueries = INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING, this::setRequestCachingEnabledForAllQueries); } public IndicesService( @@ -1746,11 +1751,11 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) { IndexSettings settings = context.indexShard().indexSettings(); // if not explicitly set in the request, use the index setting, if not, use the request if (request.requestCache() == null) { - if (settings.getValue(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING) == false) { - return false; - } else if (context.size() != 0) { + if (settings.getValue(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING) == false + || (context.size() > 0 && !requestCachingEnabledForAllQueries)) { // If no request cache query parameter and shard request cache // is enabled in settings don't cache for requests with size > 0 + // unless this is enabled via cluster setting return false; } } else if (request.requestCache() == false) { @@ -2118,4 +2123,9 @@ public RemoteStoreSettings getRemoteStoreSettings() { public CompositeIndexSettings getCompositeIndexSettings() { return this.compositeIndexSettings; } + + // Package-private for testing + void setRequestCachingEnabledForAllQueries(Boolean requestCachingEnabledForAllQueries) { + this.requestCachingEnabledForAllQueries = requestCachingEnabledForAllQueries; + } } diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java index b5350a39e8599..d2250702b48fd 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java @@ -641,25 +641,68 @@ public void testDirectoryReaderWithoutDelegatingCacheHelperNotCacheable() throws ShardSearchRequest request = mock(ShardSearchRequest.class); when(request.requestCache()).thenReturn(true); - TestSearchContext context = new TestSearchContext(indexService.getBigArrays(), indexService) { - @Override - public SearchType searchType() { - return SearchType.QUERY_THEN_FETCH; - } - }; + TestSearchContext context = getTestContext(indexService, 0); + IndexReader.CacheHelper notDelegatingCacheHelper = mock(IndexReader.CacheHelper.class); + DelegatingCacheHelper delegatingCacheHelper = mock(DelegatingCacheHelper.class); + for (boolean useDelegatingCacheHelper : new boolean[] { true, false }) { + IndexReader.CacheHelper cacheHelper = useDelegatingCacheHelper ? delegatingCacheHelper : notDelegatingCacheHelper; + setupMocksForCanCache(context, cacheHelper); + assertEquals(useDelegatingCacheHelper, indicesService.canCache(request, context)); + } + } + + public void testCanCacheSizeNonzero() { + // Size == 0 requests should always be cacheable (if they pass the other checks). + // Size > 0 requests should only be cacheable if ALLOW_SIZE_NONZERO_SETTING is true. + + final IndexService indexService = createIndex("test"); + ShardSearchRequest request = mock(ShardSearchRequest.class); + when(request.requestCache()).thenReturn(null); + + TestSearchContext sizeZeroContext = getTestContext(indexService, 0); + TestSearchContext sizeNonzeroContext = getTestContext(indexService, 10); + + // Test for an IndicesService with the default setting value of false + IndicesService indicesService = getIndicesService(); + DelegatingCacheHelper cacheHelper = mock(DelegatingCacheHelper.class); + Map expectedResultMap = Map.of(sizeZeroContext, true, sizeNonzeroContext, false); + for (Map.Entry entry : expectedResultMap.entrySet()) { + TestSearchContext context = entry.getKey(); + setupMocksForCanCache(context, cacheHelper); + assertEquals(entry.getValue(), indicesService.canCache(request, context)); + } + // Simulate the cluster setting update by manually calling setCanCacheSizeNonzeroRequests + indicesService.setRequestCachingEnabledForAllQueries(true); + expectedResultMap = Map.of(sizeZeroContext, true, sizeNonzeroContext, true); + + for (Map.Entry entry : expectedResultMap.entrySet()) { + TestSearchContext context = entry.getKey(); + setupMocksForCanCache(context, cacheHelper); + assertEquals(entry.getValue(), indicesService.canCache(request, context)); + } + } + + private void setupMocksForCanCache(TestSearchContext context, IndexReader.CacheHelper cacheHelper) { ContextIndexSearcher searcher = mock(ContextIndexSearcher.class); context.setSearcher(searcher); DirectoryReader reader = mock(DirectoryReader.class); when(searcher.getDirectoryReader()).thenReturn(reader); when(searcher.getIndexReader()).thenReturn(reader); - IndexReader.CacheHelper notDelegatingCacheHelper = mock(IndexReader.CacheHelper.class); - DelegatingCacheHelper delegatingCacheHelper = mock(DelegatingCacheHelper.class); + when(reader.getReaderCacheHelper()).thenReturn(cacheHelper); + } - for (boolean useDelegatingCacheHelper : new boolean[] { true, false }) { - IndexReader.CacheHelper cacheHelper = useDelegatingCacheHelper ? delegatingCacheHelper : notDelegatingCacheHelper; - when(reader.getReaderCacheHelper()).thenReturn(cacheHelper); - assertEquals(useDelegatingCacheHelper, indicesService.canCache(request, context)); - } + private TestSearchContext getTestContext(IndexService indexService, int size) { + return new TestSearchContext(indexService.getBigArrays(), indexService) { + @Override + public SearchType searchType() { + return SearchType.QUERY_THEN_FETCH; + } + + @Override + public int size() { + return size; + } + }; } } From a2a01f821760f7f27eaf6d30b6a4daaadda9fec8 Mon Sep 17 00:00:00 2001 From: panguixin Date: Sat, 2 Nov 2024 01:29:04 +0800 Subject: [PATCH 303/357] Support retrieving doc values of unsigned long field (#16543) * Support retrieving doc values of unsigned long field Signed-off-by: panguixin * add test Signed-off-by: panguixin * changelog Signed-off-by: panguixin * randomize test Signed-off-by: panguixin --------- Signed-off-by: panguixin --- CHANGELOG.md | 1 + .../plain/SortedNumericIndexFieldData.java | 23 +++++++++++++++++ .../index/mapper/NumberFieldTypeTests.java | 25 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84fd3810ff158..edbf7c8ed065c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Revert changes to upload remote state manifest using minimum codec version([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403)) - Ensure index templates are not applied to system indices ([#16418](https://github.com/opensearch-project/OpenSearch/pull/16418)) - Remove resource usages object from search response headers ([#16532](https://github.com/opensearch-project/OpenSearch/pull/16532)) +- Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543)) ### Security diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 0019a41e67c02..8a61d86f6f615 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -41,6 +41,7 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.time.DateUtils; import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.index.fielddata.FieldData; @@ -573,6 +574,28 @@ public final SortedBinaryDocValues getBytesValues() { return FieldData.toUnsignedString(getLongValues()); } + @Override + public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + SortedNumericDocValues values = getLongValues(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + final BigInteger value = Numbers.toUnsignedBigInteger(values.nextValue()); + return format.format(value); + } + }; + } + @Override public long ramBytesUsed() { return 0L; diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java index b27ef49303205..c06371bed9357 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java @@ -73,6 +73,7 @@ import org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.opensearch.index.mapper.NumberFieldMapper.NumberType; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.query.BitmapDocValuesQuery; import org.junit.Before; @@ -981,4 +982,28 @@ public void testBitmapQuery() throws IOException { NumberFieldType finalFt = ft; assertThrows(IllegalArgumentException.class, () -> finalFt.bitmapQuery(bitmap)); } + + public void testFetchUnsignedLongDocValues() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); + Document doc = new Document(); + final BigInteger expectedValue = randomUnsignedLong(); + doc.add(new SortedNumericDocValuesField("ul", expectedValue.longValue())); + w.addDocument(doc); + try (DirectoryReader reader = DirectoryReader.open(w)) { + final NumberFieldType ft = new NumberFieldType("ul", NumberType.UNSIGNED_LONG); + IndexNumericFieldData fielddata = (IndexNumericFieldData) ft.fielddataBuilder( + "index", + () -> { throw new UnsupportedOperationException(); } + ).build(null, null); + assertEquals(IndexNumericFieldData.NumericType.UNSIGNED_LONG, fielddata.getNumericType()); + DocValueFetcher.Leaf fetcher = fielddata.load(reader.leaves().get(0)).getLeafValueFetcher(DocValueFormat.UNSIGNED_LONG); + assertTrue(fetcher.advanceExact(0)); + assertEquals(1, fetcher.docValueCount()); + final Object value = fetcher.nextValue(); + assertTrue(value instanceof BigInteger); + assertEquals(expectedValue, value); + } + IOUtils.close(w, dir); + } } From bb131f99f41bb5f98e7b41b4a048c55dbd1af1d4 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 4 Nov 2024 11:30:54 -0500 Subject: [PATCH 304/357] Bump versions in README to 2.19.0 and 2.18.1 (#16554) Signed-off-by: Craig Perkins --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95fbac7bbecf1..6ae7e12948670 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.18.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") -[![2.17.2 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.17.2)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.17.2") +[![2.19.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.19.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.19.0") +[![2.18.1 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.18.1)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.1") [![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) From f32f5c66fa19286881eaed1160798a6883ba7d5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:02:57 -0500 Subject: [PATCH 305/357] Bump org.apache.hadoop:hadoop-minicluster from 3.4.0 to 3.4.1 in /test/fixtures/hdfs-fixture (#16550) * Bump org.apache.hadoop:hadoop-minicluster in /test/fixtures/hdfs-fixture Bumps org.apache.hadoop:hadoop-minicluster from 3.4.0 to 3.4.1. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-minicluster dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edbf7c8ed065c..52c4d5a0b478a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) +- Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index d27273f357758..18bcee8b338fc 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -37,7 +37,7 @@ versions << [ ] dependencies { - api("org.apache.hadoop:hadoop-minicluster:3.4.0") { + api("org.apache.hadoop:hadoop-minicluster:3.4.1") { exclude module: 'websocket-client' exclude module: 'jettison' exclude module: 'netty' From 4c35a2b418aa4cf14b8f49d7639d386b96752ae8 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Tue, 5 Nov 2024 05:23:32 +0800 Subject: [PATCH 306/357] fix rollover alias supports restored searchable snapshot index (#16483) Signed-off-by: kkewwei Signed-off-by: kkewwei --- CHANGELOG.md | 1 + .../alias/TransportIndicesAliasesAction.java | 4 +- .../rollover/TransportRolloverAction.java | 13 +- .../put/TransportUpdateSettingsAction.java | 7 +- .../cluster/block/ClusterBlocks.java | 26 ++- .../cluster/block/ClusterBlocksTests.java | 177 ++++++++++++++++++ 6 files changed, 215 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c4d5a0b478a..bbb30d78aa5d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Ensure index templates are not applied to system indices ([#16418](https://github.com/opensearch-project/OpenSearch/pull/16418)) - Remove resource usages object from search response headers ([#16532](https://github.com/opensearch-project/OpenSearch/pull/16532)) - Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543)) +- Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) ### Security diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 81cb3102cfcb9..42e02e9e1aff8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; -import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.AliasAction; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexAbstraction; @@ -123,7 +123,7 @@ protected ClusterBlockException checkBlock(IndicesAliasesRequest request, Cluste for (IndicesAliasesRequest.AliasActions aliasAction : request.aliasActions()) { Collections.addAll(indices, aliasAction.indices()); } - return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indices.toArray(new String[0])); + return ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, state); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index 3b11a3d82d707..28d1d14655e3b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.block.ClusterBlockException; -import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; @@ -62,8 +62,10 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -127,11 +129,10 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState request.indicesOptions().expandWildcardsClosed() ); - return state.blocks() - .indicesBlockedException( - ClusterBlockLevel.METADATA_WRITE, - indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request) - ); + return ClusterBlocks.indicesWithRemoteSnapshotBlockedException( + new HashSet<>(Arrays.asList(indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request))), + state + ); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 09cceca52ce23..779b136abef5c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; -import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; import org.opensearch.cluster.service.ClusterService; @@ -118,9 +118,8 @@ protected ClusterBlockException checkBlock(UpdateSettingsRequest request, Cluste return globalBlock; } if (request.settings().size() == 1 && // we have to allow resetting these settings otherwise users can't unblock an index - IndexMetadata.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings()) - || IndexMetadata.INDEX_READ_ONLY_SETTING.exists(request.settings()) - || IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.settings())) { + ClusterBlocks.INDEX_DATA_READ_ONLY_BLOCK_SETTINGS.stream() + .anyMatch(booleanSetting -> booleanSetting.exists(request.settings()))) { return null; } diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 615ea18315cd1..c894fa5dce714 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -33,19 +33,23 @@ package org.opensearch.cluster.block; import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.Diff; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MetadataIndexStateService; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.settings.Setting; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.VerifiableWriteable; import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexModule; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; @@ -57,6 +61,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; import static java.util.stream.Collectors.toSet; +import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; /** * Represents current cluster level blocks to block dirty operations done against the cluster. @@ -66,7 +71,11 @@ @PublicApi(since = "1.0.0") public class ClusterBlocks extends AbstractDiffable implements VerifiableWriteable { public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), Map.of()); - + public static final Set> INDEX_DATA_READ_ONLY_BLOCK_SETTINGS = Set.of( + IndexMetadata.INDEX_READ_ONLY_SETTING, + IndexMetadata.INDEX_BLOCKS_METADATA_SETTING, + IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING + ); private final Set global; private final Map> indicesBlocks; @@ -276,6 +285,21 @@ public ClusterBlockException indicesAllowReleaseResources(String[] indices) { return new ClusterBlockException(indexLevelBlocks); } + public static ClusterBlockException indicesWithRemoteSnapshotBlockedException(Collection concreteIndices, ClusterState state) { + for (String index : concreteIndices) { + if (state.blocks().indexBlocked(ClusterBlockLevel.METADATA_WRITE, index)) { + IndexMetadata indexMeta = state.metadata().index(index); + if (indexMeta != null + && (IndexModule.Type.REMOTE_SNAPSHOT.match(indexMeta.getSettings().get(INDEX_STORE_TYPE_SETTING.getKey())) == false + || ClusterBlocks.INDEX_DATA_READ_ONLY_BLOCK_SETTINGS.stream() + .anyMatch(booleanSetting -> booleanSetting.exists(indexMeta.getSettings())))) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices.toArray(new String[0])); + } + } + } + return null; + } + @Override public String toString() { if (global.isEmpty() && indices().isEmpty()) { diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java index 839e831d38b1b..47e3d0cb44cc9 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -8,12 +8,40 @@ package org.opensearch.cluster.block; +import com.carrotsearch.randomizedtesting.RandomizedTest; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.index.IndexModule; +import org.opensearch.index.IndexSettings; import org.opensearch.test.OpenSearchTestCase; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_METADATA_BLOCK; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_BLOCK; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_WRITE_BLOCK; +import static org.opensearch.cluster.metadata.IndexMetadata.REMOTE_READ_ONLY_ALLOW_DELETE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; public class ClusterBlocksTests extends OpenSearchTestCase { @@ -52,4 +80,153 @@ public void testWriteVerifiableTo() throws Exception { clusterBlocks2.writeVerifiableTo(checksumOut2); assertEquals(checksumOut.getChecksum(), checksumOut2.getChecksum()); } + + public void testGlobalBlock() { + String index = "test-000001"; + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + Set indices = new HashSet<>(); + indices.add(index); + + // no global blocks + { + stateBuilder.blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, clusterState)); + } + + // has global block + { + for (ClusterBlock block : Arrays.asList( + INDEX_READ_ONLY_BLOCK, + INDEX_READ_BLOCK, + INDEX_WRITE_BLOCK, + INDEX_METADATA_BLOCK, + INDEX_READ_ONLY_ALLOW_DELETE_BLOCK, + REMOTE_READ_ONLY_ALLOW_DELETE + )) { + stateBuilder.blocks(ClusterBlocks.builder().addGlobalBlock(block).build()); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, clusterState)); + } + } + } + + public void testIndexWithBlock() { + String index = "test-000001"; + Set indices = new HashSet<>(); + indices.add(index); + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + stateBuilder.blocks(ClusterBlocks.builder().addIndexBlock(index, IndexMetadata.INDEX_METADATA_BLOCK)); + stateBuilder.metadata(Metadata.builder().put(createIndexMetadata(index, false, null, null), false)); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNotNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, stateBuilder.build())); + } + + public void testRemoteIndexBlock() { + String remoteIndex = "remote_index"; + Set indices = new HashSet<>(); + indices.add(remoteIndex); + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + + { + IndexMetadata remoteSnapshotIndexMetadata = createIndexMetadata(remoteIndex, true, null, null); + stateBuilder.metadata(Metadata.builder().put(remoteSnapshotIndexMetadata, false)); + stateBuilder.blocks(ClusterBlocks.builder().addBlocks(remoteSnapshotIndexMetadata)); + + ClusterState clusterState = stateBuilder.build(); + assertTrue(clusterState.blocks().hasIndexBlock(remoteIndex, IndexMetadata.REMOTE_READ_ONLY_ALLOW_DELETE)); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, clusterState)); + } + + // searchable snapshot index with block + { + Setting setting = RandomizedTest.randomFrom(new ArrayList<>(ClusterBlocks.INDEX_DATA_READ_ONLY_BLOCK_SETTINGS)); + IndexMetadata remoteSnapshotIndexMetadata = createIndexMetadata(remoteIndex, true, null, setting); + stateBuilder.metadata(Metadata.builder().put(remoteSnapshotIndexMetadata, false)); + stateBuilder.blocks(ClusterBlocks.builder().addBlocks(remoteSnapshotIndexMetadata)); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNotNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, clusterState)); + } + } + + public void testRemoteIndexWithoutBlock() { + String remoteIndex = "remote_index"; + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + + String alias = "alias1"; + IndexMetadata remoteSnapshotIndexMetadata = createIndexMetadata(remoteIndex, true, alias, null); + String index = "test-000001"; + IndexMetadata indexMetadata = createIndexMetadata(index, false, alias, null); + stateBuilder.metadata(Metadata.builder().put(remoteSnapshotIndexMetadata, false).put(indexMetadata, false)); + + Set indices = new HashSet<>(); + indices.add(remoteIndex); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(indices, clusterState)); + } + + public void testRemoteIndexWithIndexBlock() { + String index = "test-000001"; + String remoteIndex = "remote_index"; + String alias = "alias1"; + { + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + IndexMetadata remoteSnapshotIndexMetadata = createIndexMetadata(remoteIndex, true, alias, null); + IndexMetadata indexMetadata = createIndexMetadata(index, false, alias, null); + stateBuilder.metadata(Metadata.builder().put(remoteSnapshotIndexMetadata, false).put(indexMetadata, false)) + .blocks(ClusterBlocks.builder().addBlocks(remoteSnapshotIndexMetadata)); + ClusterState clusterState = stateBuilder.build(); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(Collections.singleton(index), clusterState)); + clusterState.blocks(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(Collections.singleton(remoteIndex), clusterState)); + } + + { + ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName("cluster")); + Setting setting = RandomizedTest.randomFrom(new ArrayList<>(ClusterBlocks.INDEX_DATA_READ_ONLY_BLOCK_SETTINGS)); + IndexMetadata remoteSnapshotIndexMetadata = createIndexMetadata(remoteIndex, true, alias, setting); + IndexMetadata indexMetadata = createIndexMetadata(index, false, alias, null); + stateBuilder.metadata(Metadata.builder().put(remoteSnapshotIndexMetadata, false).put(indexMetadata, false)) + .blocks(ClusterBlocks.builder().addBlocks(remoteSnapshotIndexMetadata)); + ClusterState clusterState = stateBuilder.build(); + assertNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(Collections.singleton(index), clusterState)); + assertNotNull(ClusterBlocks.indicesWithRemoteSnapshotBlockedException(Collections.singleton(remoteIndex), clusterState)); + } + } + + private IndexMetadata createIndexMetadata(String index, boolean isRemoteIndex, String alias, Setting blockSetting) { + IndexMetadata.Builder builder = IndexMetadata.builder(index).settings(createIndexSettingBuilder(isRemoteIndex, blockSetting)); + if (alias != null) { + AliasMetadata.Builder aliasBuilder = AliasMetadata.builder(alias); + return builder.putAlias(aliasBuilder.build()).build(); + } + return builder.build(); + } + + private Settings.Builder createIndexSettingBuilder(boolean isRemoteIndex, Setting blockSetting) { + Settings.Builder builder = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_UUID, "abc") + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 1); + + if (isRemoteIndex) { + builder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.REMOTE_SNAPSHOT.getSettingsKey()) + .put(IndexSettings.SEARCHABLE_SNAPSHOT_REPOSITORY.getKey(), "repo") + .put(IndexSettings.SEARCHABLE_SNAPSHOT_ID_NAME.getKey(), "snapshot"); + } + if (blockSetting != null) { + builder.put(blockSetting.getKey(), true); + } + + return builder; + } } From b25e10afb9e216c547a59409d909ec1ecae101ec Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 5 Nov 2024 13:53:35 -0500 Subject: [PATCH 307/357] Ensure support of the transport-nio by security plugin (HTTP) (#16474) * Ensure support of the transport-nio by security plugin (HTTP) Signed-off-by: Andriy Redko * Add header verifier and decompressor support of secure NIO transport variant Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../ssl/SecureNetty4HttpServerTransport.java | 4 +- .../http/netty4/Netty4HttpClient.java | 2 - plugins/transport-nio/build.gradle | 5 +- ...-native-unix-common-4.1.114.Final.jar.sha1 | 1 + .../opensearch/http/nio/NioPipeliningIT.java | 4 +- .../http/nio/HttpReadWriteHandler.java | 39 +- .../org/opensearch/http/nio/NettyAdaptor.java | 20 +- .../http/nio/NioHttpServerTransport.java | 130 +++- .../org/opensearch/http/nio/ssl/SslUtils.java | 48 ++ .../opensearch/http/nio/ssl/package-info.java | 12 + .../transport/nio/NioTransportPlugin.java | 35 ++ .../opensearch/http/nio/NioHttpClient.java | 54 +- .../http/nio/NioHttpServerTransportTests.java | 12 +- .../SecureNioHttpServerTransportTests.java | 558 ++++++++++++++++++ .../src/test/resources/README.txt | 14 + .../src/test/resources/certificate.crt | 22 + .../src/test/resources/certificate.key | 28 + .../reactor/netty4/ReactorHttpClient.java | 4 +- .../SecureHttpTransportSettingsProvider.java | 10 + 20 files changed, 966 insertions(+), 37 deletions(-) create mode 100644 plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/SslUtils.java create mode 100644 plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/package-info.java create mode 100644 plugins/transport-nio/src/test/java/org/opensearch/http/nio/ssl/SecureNioHttpServerTransportTests.java create mode 100644 plugins/transport-nio/src/test/resources/README.txt create mode 100644 plugins/transport-nio/src/test/resources/certificate.crt create mode 100644 plugins/transport-nio/src/test/resources/certificate.key diff --git a/CHANGELOG.md b/CHANGELOG.md index bbb30d78aa5d0..bba62e97a49e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993)) - Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111)) +- Ensure support of the transport-nio by security plugin ([#16474](https://github.com/opensearch-project/OpenSearch/pull/16474)) - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java index 978c92870bd75..e3a6dbb4c57b5 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java @@ -67,8 +67,8 @@ * @see SecuritySSLNettyHttpServerTransport */ public class SecureNetty4HttpServerTransport extends Netty4HttpServerTransport { - public static final String REQUEST_HEADER_VERIFIER = "HeaderVerifier"; - public static final String REQUEST_DECOMPRESSOR = "RequestDecompressor"; + public static final String REQUEST_HEADER_VERIFIER = SecureHttpTransportSettingsProvider.REQUEST_HEADER_VERIFIER; + public static final String REQUEST_DECOMPRESSOR = SecureHttpTransportSettingsProvider.REQUEST_DECOMPRESSOR; private static final Logger logger = LogManager.getLogger(SecureNetty4HttpServerTransport.class); private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java index ef6b67ea44299..cf841f2e24b1e 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java @@ -315,13 +315,11 @@ private static class CountDownLatchHandlerHttp2 extends AwaitableChannelInitiali private final CountDownLatch latch; private final Collection content; - private final boolean secure; private Http2SettingsHandler settingsHandler; CountDownLatchHandlerHttp2(final CountDownLatch latch, final Collection content, final boolean secure) { this.latch = latch; this.content = content; - this.secure = secure; } @Override diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index ee557aa0efc79..c0f0150378434 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -50,6 +50,7 @@ dependencies { api "io.netty:netty-handler:${versions.netty}" api "io.netty:netty-resolver:${versions.netty}" api "io.netty:netty-transport:${versions.netty}" + api "io.netty:netty-transport-native-unix-common:${versions.netty}" } tasks.named("dependencyLicenses").configure { @@ -151,10 +152,6 @@ thirdPartyAudit { 'io.netty.internal.tcnative.SessionTicketKey', 'io.netty.internal.tcnative.SniHostNameMatcher', - // from io.netty.channel.unix (netty) - 'io.netty.channel.unix.FileDescriptor', - 'io.netty.channel.unix.UnixChannel', - 'reactor.blockhound.BlockHound$Builder', 'reactor.blockhound.integration.BlockHoundIntegration' ) diff --git a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 0000000000000..a80b9e51be74b --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-nio/src/internalClusterTest/java/org/opensearch/http/nio/NioPipeliningIT.java b/plugins/transport-nio/src/internalClusterTest/java/org/opensearch/http/nio/NioPipeliningIT.java index 4f26e8ae65259..c4541e3b1c7d3 100644 --- a/plugins/transport-nio/src/internalClusterTest/java/org/opensearch/http/nio/NioPipeliningIT.java +++ b/plugins/transport-nio/src/internalClusterTest/java/org/opensearch/http/nio/NioPipeliningIT.java @@ -61,8 +61,8 @@ public void testThatNioHttpServerSupportsPipelining() throws Exception { TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses(); TransportAddress transportAddress = randomFrom(boundAddresses); - try (NioHttpClient nettyHttpClient = new NioHttpClient()) { - Collection responses = nettyHttpClient.get(transportAddress.address(), requests); + try (NioHttpClient client = NioHttpClient.http()) { + Collection responses = client.get(transportAddress.address(), requests); assertThat(responses, hasSize(5)); Collection opaqueIds = NioHttpClient.returnOpaqueIds(responses); diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/HttpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/HttpReadWriteHandler.java index d44515f3dc727..6438cca9cc33d 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/HttpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/HttpReadWriteHandler.java @@ -32,6 +32,7 @@ package org.opensearch.http.nio; +import org.opensearch.common.Nullable; import org.opensearch.common.unit.TimeValue; import org.opensearch.http.HttpHandlingSettings; import org.opensearch.http.HttpPipelinedRequest; @@ -44,6 +45,8 @@ import org.opensearch.nio.TaskScheduler; import org.opensearch.nio.WriteOperation; +import javax.net.ssl.SSLEngine; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -58,6 +61,7 @@ import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.ssl.SslHandler; public class HttpReadWriteHandler implements NioChannelHandler { @@ -77,6 +81,28 @@ public HttpReadWriteHandler( HttpHandlingSettings settings, TaskScheduler taskScheduler, LongSupplier nanoClock + ) { + this( + nioHttpChannel, + transport, + settings, + taskScheduler, + nanoClock, + null, /* no header verifier */ + new HttpContentDecompressor(), + null /* no SSL/TLS */ + ); + } + + HttpReadWriteHandler( + NioHttpChannel nioHttpChannel, + NioHttpServerTransport transport, + HttpHandlingSettings settings, + TaskScheduler taskScheduler, + LongSupplier nanoClock, + @Nullable ChannelHandler headerVerifier, + ChannelHandler decompressor, + @Nullable SSLEngine sslEngine ) { this.nioHttpChannel = nioHttpChannel; this.transport = transport; @@ -85,6 +111,12 @@ public HttpReadWriteHandler( this.readTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(settings.getReadTimeoutMillis()); List handlers = new ArrayList<>(8); + + SslHandler sslHandler = null; + if (sslEngine != null) { + sslHandler = new SslHandler(sslEngine); + } + HttpRequestDecoder decoder = new HttpRequestDecoder( settings.getMaxInitialLineLength(), settings.getMaxHeaderSize(), @@ -92,7 +124,10 @@ public HttpReadWriteHandler( ); decoder.setCumulator(ByteToMessageDecoder.COMPOSITE_CUMULATOR); handlers.add(decoder); - handlers.add(new HttpContentDecompressor()); + if (headerVerifier != null) { + handlers.add(headerVerifier); + } + handlers.add(decompressor); handlers.add(new HttpResponseEncoder()); handlers.add(new HttpObjectAggregator(settings.getMaxContentLength())); if (settings.isCompression()) { @@ -102,7 +137,7 @@ public HttpReadWriteHandler( handlers.add(new NioHttpResponseCreator()); handlers.add(new NioHttpPipeliningHandler(transport.getLogger(), settings.getPipeliningMaxEvents())); - adaptor = new NettyAdaptor(handlers.toArray(new ChannelHandler[0])); + adaptor = new NettyAdaptor(sslHandler, handlers.toArray(new ChannelHandler[0])); adaptor.addCloseListener((v, e) -> nioHttpChannel.close()); } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NettyAdaptor.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NettyAdaptor.java index 0b7f4ee7646d1..426690b4b696d 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NettyAdaptor.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NettyAdaptor.java @@ -33,6 +33,7 @@ package org.opensearch.http.nio; import org.opensearch.ExceptionsHelper; +import org.opensearch.common.Nullable; import org.opensearch.nio.FlushOperation; import org.opensearch.nio.Page; import org.opensearch.nio.WriteOperation; @@ -49,6 +50,7 @@ import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.ssl.SslHandler; class NettyAdaptor { @@ -56,9 +58,13 @@ class NettyAdaptor { private final LinkedList flushOperations = new LinkedList<>(); NettyAdaptor(ChannelHandler... handlers) { - nettyChannel = new EmbeddedChannel(); - nettyChannel.pipeline().addLast("write_captor", new ChannelOutboundHandlerAdapter() { + this(null, handlers); + } + NettyAdaptor(@Nullable SslHandler sslHandler, ChannelHandler... handlers) { + this.nettyChannel = new EmbeddedChannel(); + + nettyChannel.pipeline().addLast("write_captor", new ChannelOutboundHandlerAdapter() { @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { // This is a little tricky. The embedded channel will complete the promise once it writes the message @@ -75,12 +81,22 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) } } }); + if (sslHandler != null) { + nettyChannel.pipeline().addAfter("write_captor", "ssl_handler", sslHandler); + } nettyChannel.pipeline().addLast(handlers); } public void close() throws Exception { assert flushOperations.isEmpty() : "Should close outbound operations before calling close"; + final SslHandler sslHandler = (SslHandler) nettyChannel.pipeline().get("ssl_handler"); + if (sslHandler != null) { + // The nettyChannel.close() or sslHandler.closeOutbound() futures will block indefinitely, + // removing the handler instead from the channel. + nettyChannel.pipeline().remove(sslHandler); + } + ChannelFuture closeFuture = nettyChannel.close(); // This should be safe as we are not a real network channel closeFuture.await(); diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java index ecf9ad9f17f87..9eca5fc87120d 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.common.Nullable; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -47,6 +48,8 @@ import org.opensearch.http.AbstractHttpServerTransport; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpServerChannel; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.nio.ssl.SslUtils; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.ChannelFactory; import org.opensearch.nio.Config; @@ -56,16 +59,28 @@ import org.opensearch.nio.NioSocketChannel; import org.opensearch.nio.ServerChannelContext; import org.opensearch.nio.SocketChannelContext; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportAdapterProvider; import org.opensearch.transport.nio.NioGroupFactory; import org.opensearch.transport.nio.PageAllocator; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLException; + import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.function.Consumer; +import java.util.stream.Collectors; + +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.codec.http.HttpContentDecompressor; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE; @@ -83,6 +98,9 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport { private static final Logger logger = LogManager.getLogger(NioHttpServerTransport.class); + public static final String REQUEST_HEADER_VERIFIER = SecureHttpTransportSettingsProvider.REQUEST_HEADER_VERIFIER; + public static final String REQUEST_DECOMPRESSOR = SecureHttpTransportSettingsProvider.REQUEST_DECOMPRESSOR; + protected final PageAllocator pageAllocator; private final NioGroupFactory nioGroupFactory; @@ -97,6 +115,34 @@ public class NioHttpServerTransport extends AbstractHttpServerTransport { private volatile NioGroup nioGroup; private ChannelFactory channelFactory; + private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider; + + public NioHttpServerTransport( + Settings settings, + NetworkService networkService, + BigArrays bigArrays, + PageCacheRecycler pageCacheRecycler, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + Dispatcher dispatcher, + NioGroupFactory nioGroupFactory, + ClusterSettings clusterSettings, + Tracer tracer + ) { + this( + settings, + networkService, + bigArrays, + pageCacheRecycler, + threadPool, + xContentRegistry, + dispatcher, + nioGroupFactory, + clusterSettings, + null, + tracer + ); + } public NioHttpServerTransport( Settings settings, @@ -108,6 +154,7 @@ public NioHttpServerTransport( Dispatcher dispatcher, NioGroupFactory nioGroupFactory, ClusterSettings clusterSettings, + @Nullable SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider, Tracer tracer ) { super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, tracer); @@ -127,6 +174,7 @@ public NioHttpServerTransport( this.reuseAddress = SETTING_HTTP_TCP_REUSE_ADDRESS.get(settings); this.tcpSendBufferSize = Math.toIntExact(SETTING_HTTP_TCP_SEND_BUFFER_SIZE.get(settings).getBytes()); this.tcpReceiveBufferSize = Math.toIntExact(SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings).getBytes()); + this.secureHttpTransportSettingsProvider = secureHttpTransportSettingsProvider; logger.debug( "using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}]," @@ -178,8 +226,8 @@ protected HttpServerChannel bind(InetSocketAddress socketAddress) throws IOExcep return httpServerChannel; } - protected ChannelFactory channelFactory() { - return new HttpChannelFactory(); + protected ChannelFactory channelFactory() throws SSLException { + return new HttpChannelFactory(secureHttpTransportSettingsProvider); } protected void acceptChannel(NioSocketChannel socketChannel) { @@ -187,8 +235,11 @@ protected void acceptChannel(NioSocketChannel socketChannel) { } private class HttpChannelFactory extends ChannelFactory { + private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider; + private final ChannelInboundHandlerAdapter headerVerifier; + private final TransportAdapterProvider decompressorProvider; - private HttpChannelFactory() { + private HttpChannelFactory(@Nullable SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider) { super( tcpNoDelay, tcpKeepAlive, @@ -199,17 +250,85 @@ private HttpChannelFactory() { tcpSendBufferSize, tcpReceiveBufferSize ); + this.secureHttpTransportSettingsProvider = secureHttpTransportSettingsProvider; + + final List headerVerifiers = getHeaderVerifiers(secureHttpTransportSettingsProvider); + final Optional> decompressorProviderOpt = getDecompressorProvider( + secureHttpTransportSettingsProvider + ); + + // There could be multiple request decompressor providers configured, using the first one + decompressorProviderOpt.ifPresent(p -> logger.debug("Using request decompressor provider: {}", p)); + + if (headerVerifiers.size() > 1) { + throw new IllegalArgumentException( + "Cannot have more than one header verifier configured, supplied " + headerVerifiers.size() + ); + } + + this.headerVerifier = headerVerifiers.isEmpty() ? null : headerVerifiers.get(0); + this.decompressorProvider = decompressorProviderOpt.orElseGet(() -> new TransportAdapterProvider() { + @Override + public String name() { + return REQUEST_DECOMPRESSOR; + } + + @Override + public Optional create(Settings settings, HttpServerTransport transport, Class adapterClass) { + return Optional.empty(); + } + }); + + } + + private List getHeaderVerifiers( + @Nullable SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider + ) { + if (secureHttpTransportSettingsProvider == null) { + return Collections.emptyList(); + } + + return secureHttpTransportSettingsProvider.getHttpTransportAdapterProviders(settings) + .stream() + .filter(p -> REQUEST_HEADER_VERIFIER.equalsIgnoreCase(p.name())) + .map(p -> p.create(settings, NioHttpServerTransport.this, ChannelInboundHandlerAdapter.class)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + private Optional> getDecompressorProvider( + @Nullable SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider + ) { + if (secureHttpTransportSettingsProvider == null) { + return Optional.empty(); + } + + return secureHttpTransportSettingsProvider.getHttpTransportAdapterProviders(settings) + .stream() + .filter(p -> REQUEST_DECOMPRESSOR.equalsIgnoreCase(p.name())) + .findFirst(); } @Override - public NioHttpChannel createChannel(NioSelector selector, SocketChannel channel, Config.Socket socketConfig) { + public NioHttpChannel createChannel(NioSelector selector, SocketChannel channel, Config.Socket socketConfig) throws IOException { + SSLEngine engine = null; + if (secureHttpTransportSettingsProvider != null) { + engine = secureHttpTransportSettingsProvider.buildSecureHttpServerEngine(settings, NioHttpServerTransport.this) + .orElseGet(SslUtils::createDefaultServerSSLEngine); + } + NioHttpChannel httpChannel = new NioHttpChannel(channel); HttpReadWriteHandler handler = new HttpReadWriteHandler( httpChannel, NioHttpServerTransport.this, handlingSettings, selector.getTaskScheduler(), - threadPool::relativeTimeInMillis + threadPool::relativeTimeInMillis, + headerVerifier, + decompressorProvider.create(settings, NioHttpServerTransport.this, ChannelInboundHandlerAdapter.class) + .orElseGet(HttpContentDecompressor::new), + engine ); Consumer exceptionHandler = (e) -> onException(httpChannel, e); SocketChannelContext context = new BytesChannelContext( @@ -244,6 +363,5 @@ public NioHttpServerChannel createServerChannel( httpServerChannel.setContext(context); return httpServerChannel; } - } } diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/SslUtils.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/SslUtils.java new file mode 100644 index 0000000000000..afd67f9799273 --- /dev/null +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/SslUtils.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.http.nio.ssl; + +import org.opensearch.OpenSearchSecurityException; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +import java.security.NoSuchAlgorithmException; + +public class SslUtils { + private static final String[] DEFAULT_SSL_PROTOCOLS = { "TLSv1.3", "TLSv1.2", "TLSv1.1" }; + + private SslUtils() { + + } + + public static SSLEngine createDefaultServerSSLEngine() { + try { + final SSLEngine engine = SSLContext.getDefault().createSSLEngine(); + engine.setEnabledProtocols(DEFAULT_SSL_PROTOCOLS); + engine.setUseClientMode(false); + return engine; + } catch (final NoSuchAlgorithmException ex) { + throw new OpenSearchSecurityException("Unable to initialize default server SSL engine", ex); + } + } + + public static SSLEngine createDefaultClientSSLEngine() { + try { + final SSLEngine engine = SSLContext.getDefault().createSSLEngine(); + engine.setEnabledProtocols(DEFAULT_SSL_PROTOCOLS); + engine.setUseClientMode(true); + return engine; + } catch (final NoSuchAlgorithmException ex) { + throw new OpenSearchSecurityException("Unable to initialize default client SSL engine", ex); + } + } +} diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/package-info.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/package-info.java new file mode 100644 index 0000000000000..a67f8247ebd4d --- /dev/null +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/ssl/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * SSL supporting utility classes + */ +package org.opensearch.http.nio.ssl; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java index d4be876867651..7707369b59120 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java @@ -47,9 +47,11 @@ import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.HttpServerTransport.Dispatcher; import org.opensearch.http.nio.NioHttpServerTransport; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -66,6 +68,7 @@ public class NioTransportPlugin extends Plugin implements NetworkPlugin { public static final String NIO_TRANSPORT_NAME = "nio-transport"; public static final String NIO_HTTP_TRANSPORT_NAME = "nio-http-transport"; + public static final String NIO_SECURE_HTTP_TRANSPORT_NAME = "nio-http-transport-secure"; private static final Logger logger = LogManager.getLogger(NioTransportPlugin.class); @@ -140,6 +143,38 @@ public Map> getHttpTransports( ); } + @Override + public Map> getSecureHttpTransports( + Settings settings, + ThreadPool threadPool, + BigArrays bigArrays, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedXContentRegistry xContentRegistry, + NetworkService networkService, + Dispatcher dispatcher, + ClusterSettings clusterSettings, + SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider, + Tracer tracer + ) { + return Collections.singletonMap( + NIO_SECURE_HTTP_TRANSPORT_NAME, + () -> new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageCacheRecycler, + threadPool, + xContentRegistry, + dispatcher, + getNioGroupFactory(settings), + clusterSettings, + secureHttpTransportSettingsProvider, + tracer + ) + ); + } + private synchronized NioGroupFactory getNioGroupFactory(Settings settings) { NioGroupFactory nioGroupFactory = groupFactory.get(); if (nioGroupFactory != null) { diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java index 45e51c6855f79..ff878eb55e411 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java @@ -71,6 +71,7 @@ import java.util.function.Consumer; import io.netty.buffer.Unpooled; +import io.netty.buffer.UnpooledByteBufAllocator; import io.netty.channel.ChannelHandler; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.DefaultFullHttpResponse; @@ -83,6 +84,10 @@ import io.netty.handler.codec.http.HttpRequestEncoder; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseDecoder; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; import static org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory; import static io.netty.handler.codec.http.HttpHeaderNames.HOST; @@ -92,7 +97,7 @@ /** * Tiny helper to send http requests over nio. */ -class NioHttpClient implements Closeable { +public class NioHttpClient implements Closeable { static Collection returnOpaqueIds(Collection responses) { List list = new ArrayList<>(responses.size()); @@ -105,9 +110,11 @@ static Collection returnOpaqueIds(Collection responses private static final Logger logger = LogManager.getLogger(NioHttpClient.class); private final NioSelectorGroup nioGroup; + private final boolean secure; - NioHttpClient() { + private NioHttpClient(final boolean secure) { try { + this.secure = secure; nioGroup = new NioSelectorGroup( daemonThreadFactory(Settings.EMPTY, "nio-http-client"), 1, @@ -118,6 +125,14 @@ static Collection returnOpaqueIds(Collection responses } } + public static NioHttpClient http() { + return new NioHttpClient(false); + } + + public static NioHttpClient https() { + return new NioHttpClient(true); + } + public Collection get(InetSocketAddress remoteAddress, String... uris) throws InterruptedException { Collection requests = new ArrayList<>(uris.length); for (int i = 0; i < uris.length; i++) { @@ -138,7 +153,8 @@ public final FullHttpResponse send(InetSocketAddress remoteAddress, FullHttpRequ public final NioSocketChannel connect(InetSocketAddress remoteAddress) { ChannelFactory factory = new ClientChannelFactory( new CountDownLatch(0), - new ArrayList<>() + new ArrayList<>(), + secure ); try { NioSocketChannel nioSocketChannel = nioGroup.openChannel(remoteAddress, factory); @@ -160,7 +176,7 @@ private synchronized Collection sendRequests(InetSocketAddress final CountDownLatch latch = new CountDownLatch(requests.size()); final Collection content = Collections.synchronizedList(new ArrayList<>(requests.size())); - ChannelFactory factory = new ClientChannelFactory(latch, content); + ChannelFactory factory = new ClientChannelFactory(latch, content, secure); NioSocketChannel nioSocketChannel = null; try { @@ -196,8 +212,9 @@ private class ClientChannelFactory extends ChannelFactory content; + private final boolean secure; - private ClientChannelFactory(CountDownLatch latch, Collection content) { + private ClientChannelFactory(CountDownLatch latch, Collection content, final boolean secure) { super( NetworkService.TCP_NO_DELAY.get(Settings.EMPTY), NetworkService.TCP_KEEP_ALIVE.get(Settings.EMPTY), @@ -210,12 +227,14 @@ private ClientChannelFactory(CountDownLatch latch, Collection ); this.latch = latch; this.content = content; + this.secure = secure; } @Override - public NioSocketChannel createChannel(NioSelector selector, java.nio.channels.SocketChannel channel, Config.Socket socketConfig) { + public NioSocketChannel createChannel(NioSelector selector, java.nio.channels.SocketChannel channel, Config.Socket socketConfig) + throws IOException { NioSocketChannel nioSocketChannel = new NioSocketChannel(channel); - HttpClientHandler handler = new HttpClientHandler(nioSocketChannel, latch, content); + HttpClientHandler handler = new HttpClientHandler(nioSocketChannel, latch, content, secure); Consumer exceptionHandler = (e) -> { latch.countDown(); onException(e); @@ -249,17 +268,34 @@ private static class HttpClientHandler implements NioChannelHandler { private final CountDownLatch latch; private final Collection content; - private HttpClientHandler(NioSocketChannel channel, CountDownLatch latch, Collection content) { + private HttpClientHandler( + NioSocketChannel channel, + CountDownLatch latch, + Collection content, + final boolean secure + ) throws IOException { this.latch = latch; this.content = content; final int maxContentLength = Math.toIntExact(new ByteSizeValue(100, ByteSizeUnit.MB).getBytes()); List handlers = new ArrayList<>(5); + + SslHandler sslHandler = null; + if (secure) { + sslHandler = new SslHandler( + SslContextBuilder.forClient() + .clientAuth(ClientAuth.NONE) + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .build() + .newEngine(UnpooledByteBufAllocator.DEFAULT) + ); + } + handlers.add(new HttpResponseDecoder()); handlers.add(new HttpRequestEncoder()); handlers.add(new HttpContentDecompressor()); handlers.add(new HttpObjectAggregator(maxContentLength)); - adaptor = new NettyAdaptor(handlers.toArray(new ChannelHandler[0])); + adaptor = new NettyAdaptor(sslHandler, handlers.toArray(new ChannelHandler[0])); adaptor.addCloseListener((v, e) -> channel.close()); } diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java index 09594673de5b2..61b42f2a6b77a 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java @@ -193,7 +193,7 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, ) { transport.start(); final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); request.headers().set(HttpHeaderNames.EXPECT, expectation); HttpUtil.setContentLength(request, contentLength); @@ -310,7 +310,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); // Test pre-flight request - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/"); request.headers().add(CorsHandler.ORIGIN, "test-cors.org"); request.headers().add(CorsHandler.ACCESS_CONTROL_REQUEST_METHOD, "POST"); @@ -327,7 +327,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th } // Test short-circuited request - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); request.headers().add(CorsHandler.ORIGIN, "google.com"); @@ -384,7 +384,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th transport.start(); final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url); request.headers().add(HttpHeaderNames.ACCEPT_ENCODING, randomFrom("deflate", "gzip")); final FullHttpResponse response = client.send(remoteAddress.address(), request); @@ -451,7 +451,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th transport.start(); final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { final String url = "/" + new String(new byte[maxInitialLineLength], Charset.forName("UTF-8")); final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url); @@ -514,7 +514,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th transport.start(); final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); - try (NioHttpClient client = new NioHttpClient()) { + try (NioHttpClient client = NioHttpClient.http()) { NioSocketChannel channel = null; try { CountDownLatch channelClosedLatch = new CountDownLatch(1); diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/ssl/SecureNioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/ssl/SecureNioHttpServerTransportTests.java new file mode 100644 index 0000000000000..1adfe0370344c --- /dev/null +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/ssl/SecureNioHttpServerTransportTests.java @@ -0,0 +1,558 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.http.nio.ssl; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchException; +import org.opensearch.common.network.NetworkAddress; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.MockBigArrays; +import org.opensearch.common.util.MockPageCacheRecycler; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.http.BindHttpException; +import org.opensearch.http.CorsHandler; +import org.opensearch.http.HttpServerTransport; +import org.opensearch.http.HttpTransportSettings; +import org.opensearch.http.NullDispatcher; +import org.opensearch.http.nio.NioHttpClient; +import org.opensearch.http.nio.NioHttpServerTransport; +import org.opensearch.nio.NioSocketChannel; +import org.opensearch.plugins.SecureHttpTransportSettingsProvider; +import org.opensearch.plugins.TransportExceptionHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.nio.NioGroupFactory; +import org.junit.After; +import org.junit.Before; + +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLException; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; +import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.handler.codec.TooLongFrameException; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpUtil; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN; +import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ENABLED; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +/** + * Tests for the secure {@link NioHttpServerTransport} class. + */ +public class SecureNioHttpServerTransportTests extends OpenSearchTestCase { + + private NetworkService networkService; + private ThreadPool threadPool; + private MockBigArrays bigArrays; + private MockPageCacheRecycler pageRecycler; + private ClusterSettings clusterSettings; + private SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider; + + @Before + public void setup() throws Exception { + networkService = new NetworkService(Collections.emptyList()); + threadPool = new TestThreadPool("test"); + pageRecycler = new MockPageCacheRecycler(Settings.EMPTY); + bigArrays = new MockBigArrays(pageRecycler, new NoneCircuitBreakerService()); + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + secureHttpTransportSettingsProvider = new SecureHttpTransportSettingsProvider() { + @Override + public Optional buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) { + return Optional.empty(); + } + + @Override + public Optional buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException { + try { + SSLEngine engine = SslContextBuilder.forServer( + SecureNioHttpServerTransportTests.class.getResourceAsStream("/certificate.crt"), + SecureNioHttpServerTransportTests.class.getResourceAsStream("/certificate.key") + ).trustManager(InsecureTrustManagerFactory.INSTANCE).build().newEngine(UnpooledByteBufAllocator.DEFAULT); + return Optional.of(engine); + } catch (final IOException ex) { + throw new SSLException(ex); + } + } + }; + } + + @After + public void shutdown() throws Exception { + if (threadPool != null) { + threadPool.shutdownNow(); + } + threadPool = null; + networkService = null; + bigArrays = null; + clusterSettings = null; + } + + /** + * Test that {@link NioHttpServerTransport} supports the "Expect: 100-continue" HTTP header + * @throws InterruptedException if the client communication with the server is interrupted + */ + public void testExpectContinueHeader() throws InterruptedException { + final Settings settings = createSettings(); + final int contentLength = randomIntBetween(1, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings).bytesAsInt()); + runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.CONTINUE); + } + + /** + * Test that {@link NioHttpServerTransport} responds to a + * 100-continue expectation with too large a content-length + * with a 413 status. + * @throws InterruptedException if the client communication with the server is interrupted + */ + public void testExpectContinueHeaderContentLengthTooLong() throws InterruptedException { + final String key = HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.getKey(); + final int maxContentLength = randomIntBetween(1, 104857600); + final Settings settings = createBuilderWithPort().put(key, maxContentLength + "b").build(); + final int contentLength = randomIntBetween(maxContentLength + 1, Integer.MAX_VALUE); + runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE); + } + + /** + * Test that {@link NioHttpServerTransport} responds to an unsupported expectation with a 417 status. + * @throws InterruptedException if the client communication with the server is interrupted + */ + public void testExpectUnsupportedExpectation() throws InterruptedException { + Settings settings = createSettings(); + runExpectHeaderTest(settings, "chocolate=yummy", 0, HttpResponseStatus.EXPECTATION_FAILED); + } + + private void runExpectHeaderTest( + final Settings settings, + final String expectation, + final int contentLength, + final HttpResponseStatus expectedStatus + ) throws InterruptedException { + final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() { + @Override + public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) { + channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done"))); + } + + @Override + public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) { + logger.error( + new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), + cause + ); + throw new AssertionError(); + } + }; + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + dispatcher, + new NioGroupFactory(settings, logger), + clusterSettings, + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + try (NioHttpClient client = NioHttpClient.https()) { + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/"); + request.headers().set(HttpHeaderNames.EXPECT, expectation); + HttpUtil.setContentLength(request, contentLength); + + final FullHttpResponse response = client.send(remoteAddress.address(), request); + try { + assertThat(response.status(), equalTo(expectedStatus)); + if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) { + final FullHttpRequest continuationRequest = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + HttpMethod.POST, + "/", + Unpooled.EMPTY_BUFFER + ); + final FullHttpResponse continuationResponse = client.send(remoteAddress.address(), continuationRequest); + try { + assertThat(continuationResponse.status(), is(HttpResponseStatus.OK)); + assertThat( + new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), + is("done") + ); + } finally { + continuationResponse.release(); + } + } + } finally { + response.release(); + } + } + } + } + + public void testBindUnavailableAddress() { + Settings initialSettings = createSettings(); + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + initialSettings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + new NullDispatcher(), + new NioGroupFactory(Settings.EMPTY, logger), + clusterSettings, + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + Settings settings = Settings.builder() + .put("http.port", remoteAddress.getPort()) + .put("network.host", remoteAddress.getAddress()) + .build(); + try ( + NioHttpServerTransport otherTransport = new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + new NullDispatcher(), + new NioGroupFactory(Settings.EMPTY, logger), + clusterSettings, + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start); + assertEquals("Failed to bind to " + NetworkAddress.format(remoteAddress.address()), bindHttpException.getMessage()); + } + } + } + + public void testCorsRequest() throws InterruptedException { + final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() { + + @Override + public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) { + logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request)); + throw new AssertionError(); + } + + @Override + public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) { + logger.error( + new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), + cause + ); + throw new AssertionError(); + } + + }; + + final Settings settings = createBuilderWithPort().put(SETTING_CORS_ENABLED.getKey(), true) + .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "test-cors.org") + .build(); + + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + dispatcher, + new NioGroupFactory(settings, logger), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + + // Test pre-flight request + try (NioHttpClient client = NioHttpClient.https()) { + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/"); + request.headers().add(CorsHandler.ORIGIN, "test-cors.org"); + request.headers().add(CorsHandler.ACCESS_CONTROL_REQUEST_METHOD, "POST"); + + final FullHttpResponse response = client.send(remoteAddress.address(), request); + try { + assertThat(response.status(), equalTo(HttpResponseStatus.OK)); + assertThat(response.headers().get(CorsHandler.ACCESS_CONTROL_ALLOW_ORIGIN), equalTo("test-cors.org")); + assertThat(response.headers().get(CorsHandler.VARY), equalTo(CorsHandler.ORIGIN)); + assertTrue(response.headers().contains(CorsHandler.DATE)); + } finally { + response.release(); + } + } + + // Test short-circuited request + try (NioHttpClient client = NioHttpClient.https()) { + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"); + request.headers().add(CorsHandler.ORIGIN, "google.com"); + + final FullHttpResponse response = client.send(remoteAddress.address(), request); + try { + assertThat(response.status(), equalTo(HttpResponseStatus.FORBIDDEN)); + } finally { + response.release(); + } + } + } + } + + public void testLargeCompressedResponse() throws InterruptedException { + final String responseString = randomAlphaOfLength(4 * 1024 * 1024); + final String url = "/thing"; + final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() { + + @Override + public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) { + if (url.equals(request.uri())) { + channel.sendResponse(new BytesRestResponse(OK, responseString)); + } else { + logger.error("--> Unexpected successful uri [{}]", request.uri()); + throw new AssertionError(); + } + } + + @Override + public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) { + logger.error( + new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), + cause + ); + throw new AssertionError(); + } + + }; + + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + Settings.EMPTY, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + dispatcher, + new NioGroupFactory(Settings.EMPTY, logger), + clusterSettings, + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + + try (NioHttpClient client = NioHttpClient.https()) { + DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url); + request.headers().add(HttpHeaderNames.ACCEPT_ENCODING, randomFrom("deflate", "gzip")); + final FullHttpResponse response = client.send(remoteAddress.address(), request); + try { + assertThat(response.status(), equalTo(HttpResponseStatus.OK)); + byte[] bytes = new byte[response.content().readableBytes()]; + response.content().readBytes(bytes); + assertThat(new String(bytes, StandardCharsets.UTF_8), equalTo(responseString)); + } finally { + response.release(); + } + } + } + } + + public void testBadRequest() throws InterruptedException { + final AtomicReference causeReference = new AtomicReference<>(); + final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() { + + @Override + public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) { + logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request)); + throw new AssertionError(); + } + + @Override + public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) { + causeReference.set(cause); + try { + final OpenSearchException e = new OpenSearchException("you sent a bad request and you should feel bad"); + channel.sendResponse(new BytesRestResponse(channel, BAD_REQUEST, e)); + } catch (final IOException e) { + throw new AssertionError(e); + } + } + + }; + + final Settings settings; + final int maxInitialLineLength; + final Setting httpMaxInitialLineLengthSetting = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH; + if (randomBoolean()) { + maxInitialLineLength = httpMaxInitialLineLengthSetting.getDefault(Settings.EMPTY).bytesAsInt(); + settings = createSettings(); + } else { + maxInitialLineLength = randomIntBetween(1, 8192); + settings = createBuilderWithPort().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build(); + } + + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + dispatcher, + new NioGroupFactory(settings, logger), + clusterSettings, + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + + try (NioHttpClient client = NioHttpClient.https()) { + final String url = "/" + new String(new byte[maxInitialLineLength], Charset.forName("UTF-8")); + final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url); + + final FullHttpResponse response = client.send(remoteAddress.address(), request); + try { + assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST)); + assertThat( + new String(response.content().array(), Charset.forName("UTF-8")), + containsString("you sent a bad request and you should feel bad") + ); + } finally { + response.release(); + } + } + } + + assertNotNull(causeReference.get()); + assertThat(causeReference.get(), instanceOf(TooLongFrameException.class)); + } + + public void testReadTimeout() throws Exception { + final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() { + + @Override + public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) { + logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request)); + throw new AssertionError("Should not have received a dispatched request"); + } + + @Override + public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) { + logger.error( + new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())), + cause + ); + throw new AssertionError("Should not have received a dispatched request"); + } + + }; + + Settings settings = createBuilderWithPort().put( + HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT.getKey(), + new TimeValue(randomIntBetween(100, 300)) + ).build(); + + try ( + NioHttpServerTransport transport = new NioHttpServerTransport( + settings, + networkService, + bigArrays, + pageRecycler, + threadPool, + xContentRegistry(), + dispatcher, + new NioGroupFactory(settings, logger), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + secureHttpTransportSettingsProvider, + NoopTracer.INSTANCE + ) + ) { + transport.start(); + final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses()); + + try (NioHttpClient client = NioHttpClient.https()) { + NioSocketChannel channel = null; + try { + CountDownLatch channelClosedLatch = new CountDownLatch(1); + channel = client.connect(remoteAddress.address()); + channel.addCloseListener((r, t) -> channelClosedLatch.countDown()); + assertTrue("Channel should be closed due to read timeout", channelClosedLatch.await(1, TimeUnit.MINUTES)); + } finally { + if (channel != null) { + channel.close(); + } + } + } + } + } + + private Settings createSettings() { + return createBuilderWithPort().build(); + } + + private Settings.Builder createBuilderWithPort() { + return Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), getPortRange()); + } +} diff --git a/plugins/transport-nio/src/test/resources/README.txt b/plugins/transport-nio/src/test/resources/README.txt new file mode 100644 index 0000000000000..a4353cee45a97 --- /dev/null +++ b/plugins/transport-nio/src/test/resources/README.txt @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# This is README describes how the certificates in this directory were created. +# This file can also be executed as a script +# + +# 1. Create certificate key + +openssl req -x509 -sha256 -newkey rsa:2048 -keyout certificate.key -out certificate.crt -days 1024 -nodes + +# 2. Export the certificate in pkcs12 format + +openssl pkcs12 -export -in certificate.crt -inkey certificate.key -out server.p12 -name netty4-secure -password pass:password + diff --git a/plugins/transport-nio/src/test/resources/certificate.crt b/plugins/transport-nio/src/test/resources/certificate.crt new file mode 100644 index 0000000000000..54c78fdbcf6de --- /dev/null +++ b/plugins/transport-nio/src/test/resources/certificate.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUddAawr5zygcd+Dcn9WVDpO4BJ7YwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTI0MDMxNDE5NDQzOVoXDTI3MDEwMjE5NDQzOVowWTELMAkGA1UEBhMCQVUxEzAR +BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 +IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAzjOKkg6Iba5zfZ8b/RYw+PGmGEfbdGuuF10Wz4Jmx/Nk4VfDLxdh +TW8VllUL2JD7uPkjABj7pW3awAbvIJ+VGbKqfBr1Nsz0mPPzhT8cfuMH/FDZgQs3 +4HuqDKr0LfC1Kw5E3WF0GVMBDNu0U+nKoeqySeYjGdxDnd3W4cqK5AnUxL0RnIny +Bw7ZuhcU55XndH/Xauro/2EpvJduDsWMdqt7ZfIf1TOmaiQHK+82yb/drVaJbczK +uTpn1Kv2bnzkQEckgq+z1dLNOOyvP2xf+nsziw5ilJe92e5GJOUJYFAlEgUAGpfD +dv6j/gTRYvdJCJItOQEQtektNCAZsoc0wwIDAQABo1MwUTAdBgNVHQ4EFgQUzHts +wIt+zhB/R4U4Do2P6rr0YhkwHwYDVR0jBBgwFoAUzHtswIt+zhB/R4U4Do2P6rr0 +YhkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAveh870jJX7vt +oLCrdugsyo79pR4f7Nr1kUy3jJrfoaoUmrjiiiHWgT22fGwp7j1GZF2mVfo8YVaK +63YNn5gB2NNZhguPOFC4AdvHRYOKRBOaOvWK8oq7BcJ//18JYI/pPnpgkYvJjqv4 +gFKaZX9qWtujHpAmKiVGs7pwYGNXfixPHRNV4owcfHMIH5dhbbqT49j94xVpjbXs +OymKtFl4kpCE/0LzKFrFcuu55Am1VLBHx2cPpHLOipgUcF5BHFlQ8AXiCMOwfPAw +d22mLB6Gt1oVEpyvQHYd3e04FetEXQ9E8T+NKWZx/8Ucf+IWBYmZBRxch6O83xgk +bAbGzqkbzQ== +-----END CERTIFICATE----- diff --git a/plugins/transport-nio/src/test/resources/certificate.key b/plugins/transport-nio/src/test/resources/certificate.key new file mode 100644 index 0000000000000..228350180935d --- /dev/null +++ b/plugins/transport-nio/src/test/resources/certificate.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOM4qSDohtrnN9 +nxv9FjD48aYYR9t0a64XXRbPgmbH82ThV8MvF2FNbxWWVQvYkPu4+SMAGPulbdrA +Bu8gn5UZsqp8GvU2zPSY8/OFPxx+4wf8UNmBCzfge6oMqvQt8LUrDkTdYXQZUwEM +27RT6cqh6rJJ5iMZ3EOd3dbhyorkCdTEvRGcifIHDtm6FxTnled0f9dq6uj/YSm8 +l24OxYx2q3tl8h/VM6ZqJAcr7zbJv92tVoltzMq5OmfUq/ZufORARySCr7PV0s04 +7K8/bF/6ezOLDmKUl73Z7kYk5QlgUCUSBQAal8N2/qP+BNFi90kIki05ARC16S00 +IBmyhzTDAgMBAAECggEAVOdiElvLjyX6xeoC00YU6hxOIMdNtHU2HMamwtDV01UD +38mMQ9KjrQelYt4n34drLrHe2IZw75/5J4JzagJrmUY47psHBwaDXItuZRokeJaw +zhLYTEs7OcKRtV+a5WOspUrdzi33aQoFb67zZG3qkpsZyFXrdBV+/fy/Iv+MCvLH +xR0jQ5mzE3cw20R7S4nddChBA/y8oKGOo6QRf2SznC1jL/+yolHvJPEn1v8AUxYm +BMPHxj1O0c4M4IxnJQ3Y5Jy9OaFMyMsFlF1hVhc/3LDDxDyOuBsVsFDicojyrRea +GKngIke0yezy7Wo4NUcp8YQhafonpWVsSJJdOUotcQKBgQD0rihFBXVtcG1d/Vy7 +FvLHrmccD56JNV744LSn2CDM7W1IulNbDUZINdCFqL91u5LpxozeE1FPY1nhwncJ +N7V7XYCaSLCuV1YJzRmUCjnzk2RyopGpzWog3f9uUFGgrk1HGbNAv99k/REya6Iu +IRSkuQhaJOj3bRXzonh0K4GjewKBgQDXvamtCioOUMSP8vq919YMkBw7F+z/fr0p +pamO8HL9eewAUg6N92JQ9kobSo/GptdmdHIjs8LqnS5C3H13GX5Qlf5GskOlCpla +V55ElaSp0gvKwWE168U7gQH4etPQAXXJrOGFaGbPj9W81hTUud7HVE88KYdfWTBo +I7TuE25tWQKBgBRjcr2Vn9xXsvVTCGgamG5lLPhcoNREGz7X0pXt34XT/vhBdnKu +331i5pZMom+YCrzqK5DRwUPBPpseTjb5amj2OKIijn5ojqXQbmI0m/GdBZC71TF2 +CXLlrMQvcy3VeGEFVjd+BYpvwAAYkfIQFZ1IQdbpHnSHpX2guzLK8UmDAoGBANUy +PIcf0EetUVHfkCIjNQfdMcjD8BTcLhsF9vWmcDxFTA9VB8ULf0D64mjt2f85yQsa +b+EQN8KZ6alxMxuLOeRxFYLPj0F9o+Y/R8wHBV48kCKhz2r1v0b6SfQ/jSm1B61x +BrxLW64qOdIOzS8bLyhUDKkrcPesr8V548aRtUKhAoGBAKlNJFd8BCGKD9Td+3dE +oP1iHTX5XZ+cQIqL0e+GMQlK4HnQP566DFZU5/GHNNAfmyxd5iSRwhTqPMHRAmOb +pqQwsyufx0dFeIBxeSO3Z6jW5h2sl4nBipZpw9bzv6EBL1xRr0SfMNZzdnf4JFzc +0htGo/VO93Z2pv8w7uGUz1nN +-----END PRIVATE KEY----- diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java index 0953e51484bd3..8d20650d76583 100644 --- a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java +++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java @@ -181,7 +181,7 @@ private List processRequestsWithBody( private List sendRequests( final InetSocketAddress remoteAddress, final Collection requests, - boolean orderer + boolean ordered ) { final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(1); try { @@ -209,7 +209,7 @@ private List sendRequests( ) .toArray(Mono[]::new); - if (orderer == false) { + if (ordered == false) { return ParallelFlux.from(monos).sequential().collectList().block(); } else { return Flux.concat(monos).flatMapSequential(r -> Mono.just(r)).collectList().block(); diff --git a/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java index ff86cbc04e240..b7a47b0f4c742 100644 --- a/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java +++ b/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java @@ -27,6 +27,16 @@ */ @ExperimentalApi public interface SecureHttpTransportSettingsProvider { + /** + * The well-known name of header verifier {@link TransportAdapterProvider} provider instance + */ + final String REQUEST_HEADER_VERIFIER = "HeaderVerifier"; + + /** + * The well-known name of request decompressor {@link TransportAdapterProvider} provider instance + */ + final String REQUEST_DECOMPRESSOR = "RequestDecompressor"; + /** * Collection of additional {@link TransportAdapterProvider}s that are specific to particular HTTP transport * @param settings settings From e07499a771afbc335e1f7f08a82f8197e5826939 Mon Sep 17 00:00:00 2001 From: Robson Araujo Date: Tue, 5 Nov 2024 11:08:13 -0800 Subject: [PATCH 308/357] Improve performance for resolving derived fields (#16564) Doing the type check before the string comparison makes it much faster to resolve derived fields. Signed-off-by: Robson Araujo --- .../opensearch/index/mapper/DefaultDerivedFieldResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java b/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java index 4dd17703b6f55..1e8ef4134a8e7 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java +++ b/server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java @@ -72,7 +72,7 @@ public Set resolvePattern(String pattern) { Set derivedFields = new HashSet<>(); if (queryShardContext != null && queryShardContext.getMapperService() != null) { for (MappedFieldType fieldType : queryShardContext.getMapperService().fieldTypes()) { - if (Regex.simpleMatch(pattern, fieldType.name()) && fieldType instanceof DerivedFieldType) { + if (fieldType instanceof DerivedFieldType && Regex.simpleMatch(pattern, fieldType.name())) { derivedFields.add(fieldType.name()); } } From 4213cc27305c37ea71e5b5a5addd17e5383e8029 Mon Sep 17 00:00:00 2001 From: Finn Date: Tue, 5 Nov 2024 13:02:07 -0800 Subject: [PATCH 309/357] Make cacheEntry.getIndexInput() privileged when fetching blobs from remote snapshot (#16544) * Make cacheEntry.getIndexInput() privileged when fetching blobs from remote store Signed-off-by: Finn Carroll * Rebase Signed-off-by: Finn Carroll * Spotless apply Signed-off-by: Finn Carroll * Clean up doPrivileged calls Signed-off-by: Finn Carroll * Comment Signed-off-by: Finn Carroll * Move fetchBlob to PrivilegedExceptionAction. Catch and unwrap IOException. Signed-off-by: Finn Carroll * Unused import Signed-off-by: Finn Carroll * Update server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java Co-authored-by: Andriy Redko Signed-off-by: Finn * Typo 'thrown'. Catch and throw unknown exception as IOException. Signed-off-by: Finn Carroll --------- Signed-off-by: Finn Carroll Signed-off-by: Finn Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../store/remote/utils/TransferManager.java | 64 +++++++++++-------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bba62e97a49e0..b94483c42c6f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove resource usages object from search response headers ([#16532](https://github.com/opensearch-project/OpenSearch/pull/16532)) - Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543)) - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) +- Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544)) ### Security diff --git a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java index 94c25202ac90c..77a8ccfafbac2 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java +++ b/server/src/main/java/org/opensearch/index/store/remote/utils/TransferManager.java @@ -24,7 +24,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.AccessController; -import java.security.PrivilegedAction; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -56,39 +57,52 @@ public TransferManager(final StreamReader streamReader, final FileCache fileCach /** * Given a blobFetchRequestList, return it's corresponding IndexInput. + * + * Note: Scripted queries/aggs may trigger a blob fetch within a new security context. + * As such the following operations require elevated permissions. + * + * cacheEntry.getIndexInput() downloads new blobs from the remote store to local fileCache. + * fileCache.compute() as inserting into the local fileCache may trigger an eviction. + * * @param blobFetchRequest to fetch * @return future of IndexInput augmented with internal caching maintenance tasks */ public IndexInput fetchBlob(BlobFetchRequest blobFetchRequest) throws IOException { - final Path key = blobFetchRequest.getFilePath(); logger.trace("fetchBlob called for {}", key.toString()); - // We need to do a privileged action here in order to fetch from remote - // and write/evict from local file cache in case this is invoked as a side - // effect of a plugin (such as a scripted search) that doesn't have the - // necessary permissions. - final CachedIndexInput cacheEntry = AccessController.doPrivileged((PrivilegedAction) () -> { - return fileCache.compute(key, (path, cachedIndexInput) -> { - if (cachedIndexInput == null || cachedIndexInput.isClosed()) { - logger.trace("Transfer Manager - IndexInput closed or not in cache"); - // Doesn't exist or is closed, either way create a new one - return new DelayedCreationCachedIndexInput(fileCache, streamReader, blobFetchRequest); - } else { - logger.trace("Transfer Manager - Already in cache"); - // already in the cache and ready to be used (open) - return cachedIndexInput; + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + CachedIndexInput cacheEntry = fileCache.compute(key, (path, cachedIndexInput) -> { + if (cachedIndexInput == null || cachedIndexInput.isClosed()) { + logger.trace("Transfer Manager - IndexInput closed or not in cache"); + // Doesn't exist or is closed, either way create a new one + return new DelayedCreationCachedIndexInput(fileCache, streamReader, blobFetchRequest); + } else { + logger.trace("Transfer Manager - Already in cache"); + // already in the cache and ready to be used (open) + return cachedIndexInput; + } + }); + + // Cache entry was either retrieved from the cache or newly added, either + // way the reference count has been incremented by one. We can only + // decrement this reference _after_ creating the clone to be returned. + try { + return cacheEntry.getIndexInput().clone(); + } finally { + fileCache.decRef(key); } }); - }); - - // Cache entry was either retrieved from the cache or newly added, either - // way the reference count has been incremented by one. We can only - // decrement this reference _after_ creating the clone to be returned. - try { - return cacheEntry.getIndexInput().clone(); - } finally { - fileCache.decRef(key); + } catch (PrivilegedActionException e) { + final Exception cause = e.getException(); + if (cause instanceof IOException) { + throw (IOException) cause; + } else if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } else { + throw new IOException(cause); + } } } From 034bd2b6483c180b4a4439d62452cc50198c37fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:02:00 +0800 Subject: [PATCH 310/357] Bump com.google.apis:google-api-services-compute from v1-rev20241015-2.0.0 to v1-rev20241021-2.0.0 in /plugins/discovery-gce (#16548) * Bump com.google.apis:google-api-services-compute Bumps com.google.apis:google-api-services-compute from v1-rev20241015-2.0.0 to v1-rev20241021-2.0.0. --- updated-dependencies: - dependency-name: com.google.apis:google-api-services-compute dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/discovery-gce/build.gradle | 2 +- .../google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 | 1 - .../google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index b94483c42c6f4..6d9cf50d0da5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) -- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241015-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502)) +- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 4e05544a33f1d..5f4670357f927 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -18,7 +18,7 @@ opensearchplugin { } dependencies { - api "com.google.apis:google-api-services-compute:v1-rev20241015-2.0.0" + api "com.google.apis:google-api-services-compute:v1-rev20241021-2.0.0" api "com.google.api-client:google-api-client:1.35.2" api "com.google.oauth-client:google-oauth-client:1.36.0" api "com.google.http-client:google-http-client:${versions.google_http_client}" diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 deleted file mode 100644 index 1de9a570242fd..0000000000000 --- a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241015-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -83d293916d59ced480e48fd8c0aefb643e27566c \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..309d10035f35a --- /dev/null +++ b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 @@ -0,0 +1 @@ +cc3bd864ec5ac819699ea24a64109bfda42cb55c \ No newline at end of file From 9f790ee1e89063fd38501e64faf1df7109b3f4ec Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 6 Nov 2024 11:48:05 -0800 Subject: [PATCH 311/357] Fix non-x64 bwc build targets (#16575) There were a few issues here: the '-x64' suffix was being unconditionally appeneded, debian uses underscores not hyphens, and the rpm target uses the '.86_64' suffix. Signed-off-by: Andrew Ross --- .../InternalDistributionBwcSetupPlugin.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 0502280cb69ad..846c7e0d46b70 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -181,15 +181,19 @@ private static List resolveArchiveProjects(File checkoutDir if (name.contains("zip") || name.contains("tar")) { int index = name.lastIndexOf('-'); String baseName = name.substring(0, index); - classifier = "-" + baseName + "-x64"; + classifier = "-" + baseName; + // The x64 variants do not have the architecture built into the task name, so it needs to be appended + if (name.equals("darwin-tar") || name.equals("linux-tar") || name.equals("windows-zip")) { + classifier += "-x64"; + } extension = name.substring(index + 1); if (extension.equals("tar")) { extension += ".gz"; } } else if (name.contains("deb")) { - classifier = "-amd64"; + classifier = "_amd64"; } else if (name.contains("rpm")) { - classifier = "-x64"; + classifier = ".x86_64"; } } else { extension = name.substring(4); @@ -256,9 +260,21 @@ private static class DistributionProject { this.name = name; this.projectPath = baseDir + "/" + name; if (version.onOrAfter("1.1.0")) { + // Deb uses underscores (I don't know why...): + // https://github.com/opensearch-project/OpenSearch/blob/f6d9a86f0e2e8241fd58b7e8b6cdeaf931b5108f/distribution/packages/build.gradle#L139 + final String separator = name.equals("deb") ? "_" : "-"; this.distFile = new File( checkoutDir, - baseDir + "/" + name + "/build/distributions/opensearch-min-" + version + "-SNAPSHOT" + classifier + "." + extension + baseDir + + "/" + + name + + "/build/distributions/opensearch-min" + + separator + + version + + "-SNAPSHOT" + + classifier + + "." + + extension ); } else { this.distFile = new File( From aa5c39bbbce5bfcb06f4892ff5d6ccaea79126b2 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 7 Nov 2024 10:44:40 -0500 Subject: [PATCH 312/357] Detect Breaking Changes check does not fail on new method added to an @PublicApi interface (#16585) Signed-off-by: Andriy Redko --- server/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/server/build.gradle b/server/build.gradle index c19e171c90f96..d3c7d4089125c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -414,6 +414,7 @@ tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) { onlyModified = true failOnModification = true ignoreMissingClasses = true + failOnSourceIncompatibility = true annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi'] annotationExcludes = ['@org.opensearch.common.annotation.InternalApi'] txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt") From 9b7681c1e56db5dd61787bd6f1ff9015781a8717 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Thu, 7 Nov 2024 10:29:42 -0800 Subject: [PATCH 313/357] Make IndexStoreListener a pluggable interface (#16583) Signed-off-by: Jay Deng --- CHANGELOG.md | 3 +- .../org/opensearch/env/NodeEnvironment.java | 15 +--- .../index/store/IndexStoreListener.java | 73 +++++++++++++++++++ .../remote/filecache/FileCacheCleaner.java | 3 +- .../main/java/org/opensearch/node/Node.java | 22 +++++- .../opensearch/plugins/IndexStorePlugin.java | 9 +++ .../opensearch/env/NodeEnvironmentTests.java | 42 ++++++++--- 7 files changed, 138 insertions(+), 29 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/store/IndexStoreListener.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d9cf50d0da5a..60535b2cca895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) -- Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483/files)) +- Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) +- Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java index 709c0eba4f57f..5c6e44d063dd7 100644 --- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java @@ -71,6 +71,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.FsDirectoryFactory; +import org.opensearch.index.store.IndexStoreListener; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.fs.FsProbe; import org.opensearch.monitor.jvm.JvmInfo; @@ -1412,18 +1413,4 @@ private static void tryWriteTempFile(Path path) throws IOException { } } } - - /** - * A listener that is executed on per-index and per-shard store events, like deleting shard path - * - * @opensearch.internal - */ - public interface IndexStoreListener { - default void beforeShardPathDeleted(ShardId shardId, IndexSettings indexSettings, NodeEnvironment env) {} - - default void beforeIndexPathDeleted(Index index, IndexSettings indexSettings, NodeEnvironment env) {} - - IndexStoreListener EMPTY = new IndexStoreListener() { - }; - } } diff --git a/server/src/main/java/org/opensearch/index/store/IndexStoreListener.java b/server/src/main/java/org/opensearch/index/store/IndexStoreListener.java new file mode 100644 index 0000000000000..5a8dd28d43bbc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/IndexStoreListener.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.index.IndexSettings; + +import java.util.Collections; +import java.util.List; + +/** + * A listener that is executed on per-index and per-shard store events, like deleting shard path + * + * @opensearch.api + */ +@PublicApi(since = "2.19.0") +public interface IndexStoreListener { + default void beforeShardPathDeleted(ShardId shardId, IndexSettings indexSettings, NodeEnvironment env) {} + + default void beforeIndexPathDeleted(Index index, IndexSettings indexSettings, NodeEnvironment env) {} + + IndexStoreListener EMPTY = new IndexStoreListener() { + }; + + /** + * A Composite listener that multiplexes calls to each of the listeners methods. + * + * @opensearch.api + */ + @PublicApi(since = "2.19.0") + final class CompositeIndexStoreListener implements IndexStoreListener { + private final List listeners; + private final static Logger logger = LogManager.getLogger(CompositeIndexStoreListener.class); + + public CompositeIndexStoreListener(List listeners) { + this.listeners = Collections.unmodifiableList(listeners); + } + + @Override + public void beforeShardPathDeleted(ShardId shardId, IndexSettings indexSettings, NodeEnvironment env) { + for (IndexStoreListener listener : listeners) { + try { + listener.beforeShardPathDeleted(shardId, indexSettings, env); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("beforeShardPathDeleted listener [{}] failed", listener), e); + } + } + } + + @Override + public void beforeIndexPathDeleted(Index index, IndexSettings indexSettings, NodeEnvironment env) { + for (IndexStoreListener listener : listeners) { + try { + listener.beforeIndexPathDeleted(index, indexSettings, env); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("beforeIndexPathDeleted listener [{}] failed", listener), e); + } + } + } + } +} diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java index 0261ab24dfa7a..3cdd41b94a5e9 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java @@ -18,6 +18,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.IndexStoreListener; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -33,7 +34,7 @@ * * @opensearch.internal */ -public class FileCacheCleaner implements NodeEnvironment.IndexStoreListener { +public class FileCacheCleaner implements IndexStoreListener { private static final Logger logger = LogManager.getLogger(FileCacheCleaner.class); private final Provider fileCacheProvider; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index e74fca60b0201..c78ee6711dcda 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -157,6 +157,7 @@ import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; +import org.opensearch.index.store.IndexStoreListener; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheCleaner; @@ -548,10 +549,27 @@ protected Node( */ this.environment = new Environment(settings, initialEnvironment.configDir(), Node.NODE_LOCAL_STORAGE_SETTING.get(settings)); Environment.assertEquivalent(initialEnvironment, this.environment); + Stream indexStoreListenerStream = pluginsService.filterPlugins(IndexStorePlugin.class) + .stream() + .map(IndexStorePlugin::getIndexStoreListener) + .filter(Optional::isPresent) + .map(Optional::get); + // FileCache is only initialized on search nodes, so we only create FileCacheCleaner on search nodes as well if (DiscoveryNode.isSearchNode(settings) == false) { - nodeEnvironment = new NodeEnvironment(tmpSettings, environment); + nodeEnvironment = new NodeEnvironment( + settings, + environment, + new IndexStoreListener.CompositeIndexStoreListener(indexStoreListenerStream.collect(Collectors.toList())) + ); } else { - nodeEnvironment = new NodeEnvironment(settings, environment, new FileCacheCleaner(this::fileCache)); + nodeEnvironment = new NodeEnvironment( + settings, + environment, + new IndexStoreListener.CompositeIndexStoreListener( + Stream.concat(indexStoreListenerStream, Stream.of(new FileCacheCleaner(this::fileCache))) + .collect(Collectors.toList()) + ) + ); } logger.info( "node name [{}], node ID [{}], cluster name [{}], roles {}", diff --git a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java index ebd5717a00319..f0df8a122ed7d 100644 --- a/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/IndexStorePlugin.java @@ -39,11 +39,13 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.IndexStoreListener; import org.opensearch.indices.recovery.RecoveryState; import java.io.IOException; import java.util.Collections; import java.util.Map; +import java.util.Optional; /** * A plugin that provides alternative directory implementations. @@ -105,4 +107,11 @@ interface RecoveryStateFactory { default Map getRecoveryStateFactories() { return Collections.emptyMap(); } + + /** + * The {@link IndexStoreListener}s for this plugin which are triggered upon shard/index path deletion + */ + default Optional getIndexStoreListener() { + return Optional.empty(); + } } diff --git a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java index 962eb743dca6e..3ee9e859c198f 100644 --- a/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/opensearch/env/NodeEnvironmentTests.java @@ -45,6 +45,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.gateway.MetadataStateFormat; import org.opensearch.index.IndexSettings; +import org.opensearch.index.store.IndexStoreListener; import org.opensearch.node.Node; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.NodeRoles; @@ -360,24 +361,39 @@ protected void doRun() throws Exception { } public void testIndexStoreListener() throws Exception { - final AtomicInteger shardCounter = new AtomicInteger(0); - final AtomicInteger indexCounter = new AtomicInteger(0); + final AtomicInteger shardCounter1 = new AtomicInteger(0); + final AtomicInteger shardCounter2 = new AtomicInteger(0); + final AtomicInteger indexCounter1 = new AtomicInteger(0); + final AtomicInteger indexCounter2 = new AtomicInteger(0); final Index index = new Index("foo", "fooUUID"); final ShardId shardId = new ShardId(index, 0); - final NodeEnvironment.IndexStoreListener listener = new NodeEnvironment.IndexStoreListener() { + final IndexStoreListener listener1 = new IndexStoreListener() { @Override public void beforeShardPathDeleted(ShardId inShardId, IndexSettings indexSettings, NodeEnvironment env) { assertEquals(shardId, inShardId); - shardCounter.incrementAndGet(); + shardCounter1.incrementAndGet(); } @Override public void beforeIndexPathDeleted(Index inIndex, IndexSettings indexSettings, NodeEnvironment env) { assertEquals(index, inIndex); - indexCounter.incrementAndGet(); + indexCounter1.incrementAndGet(); } }; - final NodeEnvironment env = newNodeEnvironment(listener); + final IndexStoreListener listener2 = new IndexStoreListener() { + @Override + public void beforeShardPathDeleted(ShardId inShardId, IndexSettings indexSettings, NodeEnvironment env) { + assertEquals(shardId, inShardId); + shardCounter2.incrementAndGet(); + } + + @Override + public void beforeIndexPathDeleted(Index inIndex, IndexSettings indexSettings, NodeEnvironment env) { + assertEquals(index, inIndex); + indexCounter2.incrementAndGet(); + } + }; + final NodeEnvironment env = newNodeEnvironment(new IndexStoreListener.CompositeIndexStoreListener(List.of(listener1, listener2))); for (Path path : env.indexPaths(index)) { Files.createDirectories(path.resolve("0")); @@ -386,26 +402,30 @@ public void beforeIndexPathDeleted(Index inIndex, IndexSettings indexSettings, N for (Path path : env.indexPaths(index)) { assertTrue(Files.exists(path.resolve("0"))); } - assertEquals(0, shardCounter.get()); + assertEquals(0, shardCounter1.get()); + assertEquals(0, shardCounter2.get()); env.deleteShardDirectorySafe(new ShardId(index, 0), idxSettings); for (Path path : env.indexPaths(index)) { assertFalse(Files.exists(path.resolve("0"))); } - assertEquals(1, shardCounter.get()); + assertEquals(1, shardCounter1.get()); + assertEquals(1, shardCounter2.get()); for (Path path : env.indexPaths(index)) { assertTrue(Files.exists(path)); } - assertEquals(0, indexCounter.get()); + assertEquals(0, indexCounter1.get()); + assertEquals(0, indexCounter2.get()); env.deleteIndexDirectorySafe(index, 5000, idxSettings); for (Path path : env.indexPaths(index)) { assertFalse(Files.exists(path)); } - assertEquals(1, indexCounter.get()); + assertEquals(1, indexCounter1.get()); + assertEquals(1, indexCounter2.get()); assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty()); env.close(); } @@ -680,7 +700,7 @@ public NodeEnvironment newNodeEnvironment() throws IOException { return newNodeEnvironment(Settings.EMPTY); } - public NodeEnvironment newNodeEnvironment(NodeEnvironment.IndexStoreListener listener) throws IOException { + public NodeEnvironment newNodeEnvironment(IndexStoreListener listener) throws IOException { Settings build = buildEnvSettings(Settings.EMPTY); return new NodeEnvironment(build, TestEnvironment.newEnvironment(build), listener); } From 5909e1ad30ae00476d121536a5cb415eafc15a9e Mon Sep 17 00:00:00 2001 From: "mend-for-github-com[bot]" <50673670+mend-for-github-com[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:56:58 -0500 Subject: [PATCH 314/357] Update dependency org.apache.zookeeper:zookeeper to v3.9.3 (#16593) Co-authored-by: mend-for-github-com[bot] <50673670+mend-for-github-com[bot]@users.noreply.github.com> --- test/fixtures/hdfs-fixture/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 18bcee8b338fc..8a402879970d7 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -71,7 +71,7 @@ dependencies { api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" api "org.eclipse.jetty:jetty-server:${versions.jetty}" api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}" - api 'org.apache.zookeeper:zookeeper:3.9.2' + api 'org.apache.zookeeper:zookeeper:3.9.3' api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" api "ch.qos.logback:logback-core:1.5.12" From e68838819710d7040cf2b591590285f1b86f0da0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:42:53 -0500 Subject: [PATCH 315/357] [AUTO] [main] Add bwc version 2.18.1. (#16573) * Add bwc version 2.18.1 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update Version.java Signed-off-by: Andriy Redko --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Andriy Redko Co-authored-by: opensearch-ci-bot <83309141+opensearch-ci-bot@users.noreply.github.com> Co-authored-by: Andriy Redko --- .ci/bwcVersions | 1 + libs/core/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 17bb5a7df9b21..d1b4e4c509cb9 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -41,4 +41,5 @@ BWC_VERSION: - "2.17.1" - "2.17.2" - "2.18.0" + - "2.18.1" - "2.19.0" \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 4d685e3bc654a..ec0a18dbbf882 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -112,6 +112,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1); public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0); + public static final Version V_2_18_1 = new Version(2180199, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version V_2_19_0 = new Version(2190099, org.apache.lucene.util.Version.LUCENE_9_12_0); public static final Version CURRENT = V_3_0_0; From 10873f16e43780dbac4bf879e3324285461581cc Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:45:43 +0530 Subject: [PATCH 316/357] Increase segrep pressure checkpoint default limit to 10 (#16577) Signed-off-by: Gaurav Bafna --- CHANGELOG.md | 1 + .../opensearch/index/SegmentReplicationPressureService.java | 2 +- .../index/SegmentReplicationPressureServiceTests.java | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60535b2cca895..a0529d8fa6b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) - Add Setting to adjust the primary constraint weights ([#16471](https://github.com/opensearch-project/OpenSearch/pull/16471)) - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) +- Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files)) - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java index 297fe093f7f4e..03b162a9c1755 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationPressureService.java @@ -60,7 +60,7 @@ public class SegmentReplicationPressureService implements Closeable { public static final Setting MAX_INDEXING_CHECKPOINTS = Setting.intSetting( "segrep.pressure.checkpoint.limit", - 4, + 30, 1, Setting.Property.Dynamic, Setting.Property.NodeScope diff --git a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java index a9725f638cc53..166c0e16bfe8b 100644 --- a/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/SegmentReplicationPressureServiceTests.java @@ -33,6 +33,7 @@ import org.mockito.stubbing.Answer; import static java.util.Arrays.asList; +import static org.opensearch.index.SegmentReplicationPressureService.MAX_INDEXING_CHECKPOINTS; import static org.opensearch.index.SegmentReplicationPressureService.MAX_REPLICATION_LIMIT_STALE_REPLICA_SETTING; import static org.opensearch.index.SegmentReplicationPressureService.MAX_REPLICATION_TIME_BACKPRESSURE_SETTING; import static org.opensearch.index.SegmentReplicationPressureService.SEGMENT_REPLICATION_INDEXING_PRESSURE_ENABLED; @@ -53,6 +54,7 @@ public class SegmentReplicationPressureServiceTests extends OpenSearchIndexLevel .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SEGMENT_REPLICATION_INDEXING_PRESSURE_ENABLED.getKey(), true) .put(MAX_REPLICATION_TIME_BACKPRESSURE_SETTING.getKey(), TimeValue.timeValueSeconds(5)) + .put(MAX_INDEXING_CHECKPOINTS.getKey(), 4) .build(); public void testIsSegrepLimitBreached() throws Exception { @@ -200,6 +202,7 @@ public void testFailStaleReplicaTask() throws Exception { .put(SEGMENT_REPLICATION_INDEXING_PRESSURE_ENABLED.getKey(), true) .put(MAX_REPLICATION_TIME_BACKPRESSURE_SETTING.getKey(), TimeValue.timeValueMillis(10)) .put(MAX_REPLICATION_LIMIT_STALE_REPLICA_SETTING.getKey(), TimeValue.timeValueMillis(20)) + .put(MAX_INDEXING_CHECKPOINTS.getKey(), 4) .build(); try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { From 607a08e465014a9f8615ee30f3d5d402284ea9ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 09:56:42 -0500 Subject: [PATCH 317/357] Bump lycheeverse/lychee-action from 2.0.2 to 2.1.0 (#16610) * Bump lycheeverse/lychee-action from 2.0.2 to 2.1.0 Bumps [lycheeverse/lychee-action](https://github.com/lycheeverse/lychee-action) from 2.0.2 to 2.1.0. - [Release notes](https://github.com/lycheeverse/lychee-action/releases) - [Commits](https://github.com/lycheeverse/lychee-action/compare/v2.0.2...v2.1.0) --- updated-dependencies: - dependency-name: lycheeverse/lychee-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/links.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index cadbe71bb6ea8..3697750dab97a 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - name: lychee Link Checker id: lychee - uses: lycheeverse/lychee-action@v2.0.2 + uses: lycheeverse/lychee-action@v2.1.0 with: args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes fail: true diff --git a/CHANGELOG.md b/CHANGELOG.md index a0529d8fa6b63..e95a95990beaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) +- Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) ### Changed From 6e34a8024a2b884143f101f03e6ebffab9eed6e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 12:25:47 -0500 Subject: [PATCH 318/357] Bump me.champeau.gradle.japicmp from 0.4.4 to 0.4.5 in /server (#16614) * Bump me.champeau.gradle.japicmp from 0.4.4 to 0.4.5 in /server Bumps me.champeau.gradle.japicmp from 0.4.4 to 0.4.5. --- updated-dependencies: - dependency-name: me.champeau.gradle.japicmp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + server/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e95a95990beaf..d3086096cb8f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) +- Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) ### Changed diff --git a/server/build.gradle b/server/build.gradle index d3c7d4089125c..f1679ccfbec30 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -36,7 +36,7 @@ plugins { id('opensearch.publish') id('opensearch.internal-cluster-test') id('opensearch.optional-dependencies') - id('me.champeau.gradle.japicmp') version '0.4.4' + id('me.champeau.gradle.japicmp') version '0.4.5' } publishing { From 46ded36a27e276a1644a65b8c8ca439e5cbe4b91 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Mon, 11 Nov 2024 10:25:32 -0800 Subject: [PATCH 319/357] Adds Integration Tests for Search Pipeline (#16561) * Adds Integration Tests for Search Pipeline Signed-off-by: Owais * Addressed comments Signed-off-by: Owais --------- Signed-off-by: Owais --- .../common/SearchPipelineCommonIT.java | 186 +++++++++++++++--- 1 file changed, 155 insertions(+), 31 deletions(-) diff --git a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java index b8b0798812df1..35d748bc2d06e 100644 --- a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java +++ b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java @@ -8,41 +8,187 @@ package org.opensearch.search.pipeline.common; -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.DeleteSearchPipelineRequest; +import org.opensearch.action.search.GetSearchPipelineRequest; +import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.ingest.PipelineConfiguration; import org.opensearch.plugins.Plugin; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; @OpenSearchIntegTestCase.SuiteScopeTestCase public class SearchPipelineCommonIT extends OpenSearchIntegTestCase { + private static final String TEST_INDEX = "myindex"; + private static final String PIPELINE_NAME = "test_pipeline"; + @Override protected Collection> nodePlugins() { return List.of(SearchPipelineCommonModulePlugin.class); } + @Before + public void setup() throws Exception { + createIndex(TEST_INDEX); + + IndexRequest doc1 = new IndexRequest(TEST_INDEX).id("doc1").source(Map.of("field", "value")); + IndexRequest doc2 = new IndexRequest(TEST_INDEX).id("doc2").source(Map.of("field", "something else")); + + IndexResponse ir = client().index(doc1).actionGet(); + assertSame(RestStatus.CREATED, ir.status()); + ir = client().index(doc2).actionGet(); + assertSame(RestStatus.CREATED, ir.status()); + + RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(TEST_INDEX)).actionGet(); + assertSame(RestStatus.OK, refRsp.getStatus()); + } + + @After + public void cleanup() throws Exception { + internalCluster().wipeIndices(TEST_INDEX); + } + public void testFilterQuery() { // Create a pipeline with a filter_query processor. - String pipelineName = "foo"; + createPipeline(); + + // Search without the pipeline. Should see both documents. + SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); + SearchResponse rsp = client().search(req).actionGet(); + assertEquals(2, rsp.getHits().getTotalHits().value); + + // Search with the pipeline. Should only see document with "field":"value". + req.pipeline(PIPELINE_NAME); + rsp = client().search(req).actionGet(); + assertEquals(1, rsp.getHits().getTotalHits().value); + + // Clean up. + deletePipeline(); + } + + public void testSearchWithTemporaryPipeline() throws Exception { + + // Search without the pipeline. Should see both documents. + SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); + SearchResponse rsp = client().search(req).actionGet(); + assertEquals(2, rsp.getHits().getTotalHits().value); + + // Search with temporary pipeline + Map pipelineSourceMap = new HashMap<>(); + Map requestProcessorConfig = new HashMap<>(); + + Map filterQuery = new HashMap<>(); + filterQuery.put("query", Map.of("term", Map.of("field", "value"))); + requestProcessorConfig.put("filter_query", filterQuery); + pipelineSourceMap.put("request_processors", List.of(requestProcessorConfig)); + + req = new SearchRequest(TEST_INDEX).source( + new SearchSourceBuilder().query(new MatchAllQueryBuilder()).searchPipelineSource(pipelineSourceMap) + ); + + SearchResponse rspWithTempPipeline = client().search(req).actionGet(); + assertEquals(1, rspWithTempPipeline.getHits().getTotalHits().value); + } + + public void testSearchWithDefaultPipeline() throws Exception { + // Create pipeline + createPipeline(); + + // Search without the pipeline. Should see both documents. + SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); + SearchResponse rsp = client().search(req).actionGet(); + assertEquals(2, rsp.getHits().getTotalHits().value); + + // Set pipeline as default for the index + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX); + updateSettingsRequest.settings(Settings.builder().put("index.search.default_pipeline", PIPELINE_NAME)); + AcknowledgedResponse updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet(); + assertTrue(updateSettingsResponse.isAcknowledged()); + + // Search with the default pipeline. Should only see document with "field":"value". + rsp = client().search(req).actionGet(); + assertEquals(1, rsp.getHits().getTotalHits().value); + + // Clean up: Remove default pipeline setting + updateSettingsRequest = new UpdateSettingsRequest(TEST_INDEX); + updateSettingsRequest.settings(Settings.builder().putNull("index.search.default_pipeline")); + updateSettingsResponse = client().admin().indices().updateSettings(updateSettingsRequest).actionGet(); + assertTrue(updateSettingsResponse.isAcknowledged()); + + // Clean up. + deletePipeline(); + } + + public void testUpdateSearchPipeline() throws Exception { + // Create initial pipeline + createPipeline(); + + // Verify initial pipeline + SearchRequest req = new SearchRequest(TEST_INDEX).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); + req.pipeline(PIPELINE_NAME); + SearchResponse initialRsp = client().search(req).actionGet(); + assertEquals(1, initialRsp.getHits().getTotalHits().value); + + BytesReference pipelineConfig = new BytesArray( + "{" + + "\"description\": \"Updated pipeline\"," + + "\"request_processors\": [" + + "{" + + "\"filter_query\" : {" + + "\"query\": {" + + "\"term\" : {" + + "\"field\" : \"something else\"" + + "}" + + "}" + + "}" + + "}" + + "]" + + "}" + ); + + PipelineConfiguration pipeline = new PipelineConfiguration(PIPELINE_NAME, pipelineConfig, MediaTypeRegistry.JSON); + + // Update pipeline + PutSearchPipelineRequest updateRequest = new PutSearchPipelineRequest(pipeline.getId(), pipelineConfig, MediaTypeRegistry.JSON); + AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(updateRequest).actionGet(); + assertTrue(ackRsp.isAcknowledged()); + + // Verify pipeline description + GetSearchPipelineResponse getPipelineResponse = client().admin() + .cluster() + .getSearchPipeline(new GetSearchPipelineRequest(PIPELINE_NAME)) + .actionGet(); + assertEquals(PIPELINE_NAME, getPipelineResponse.pipelines().get(0).getId()); + assertEquals(pipeline.getConfigAsMap(), getPipelineResponse.pipelines().get(0).getConfigAsMap()); + // Clean up. + deletePipeline(); + } + + private void createPipeline() { PutSearchPipelineRequest putSearchPipelineRequest = new PutSearchPipelineRequest( - pipelineName, + PIPELINE_NAME, new BytesArray( "{" + "\"request_processors\": [" @@ -62,35 +208,13 @@ public void testFilterQuery() { ); AcknowledgedResponse ackRsp = client().admin().cluster().putSearchPipeline(putSearchPipelineRequest).actionGet(); assertTrue(ackRsp.isAcknowledged()); + } - // Index some documents. - String indexName = "myindex"; - IndexRequest doc1 = new IndexRequest(indexName).id("doc1").source(Map.of("field", "value")); - IndexRequest doc2 = new IndexRequest(indexName).id("doc2").source(Map.of("field", "something else")); - - IndexResponse ir = client().index(doc1).actionGet(); - assertSame(RestStatus.CREATED, ir.status()); - ir = client().index(doc2).actionGet(); - assertSame(RestStatus.CREATED, ir.status()); - - // Refresh so the documents are visible to search. - RefreshResponse refRsp = client().admin().indices().refresh(new RefreshRequest(indexName)).actionGet(); - assertSame(RestStatus.OK, refRsp.getStatus()); - - // Search without the pipeline. Should see both documents. - SearchRequest req = new SearchRequest(indexName).source(new SearchSourceBuilder().query(new MatchAllQueryBuilder())); - SearchResponse rsp = client().search(req).actionGet(); - assertEquals(2, rsp.getHits().getTotalHits().value); - - // Search with the pipeline. Should only see document with "field":"value". - req.pipeline(pipelineName); - rsp = client().search(req).actionGet(); - assertEquals(1, rsp.getHits().getTotalHits().value); - - // Clean up. - ackRsp = client().admin().cluster().deleteSearchPipeline(new DeleteSearchPipelineRequest(pipelineName)).actionGet(); - assertTrue(ackRsp.isAcknowledged()); - ackRsp = client().admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet(); + private void deletePipeline() { + AcknowledgedResponse ackRsp = client().admin() + .cluster() + .deleteSearchPipeline(new DeleteSearchPipelineRequest(PIPELINE_NAME)) + .actionGet(); assertTrue(ackRsp.isAcknowledged()); } } From c9edb48c1858164cf86c2cc1c7f7d493e01a62bc Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 11 Nov 2024 10:59:30 -0800 Subject: [PATCH 320/357] Add a flag in QueryShardContext to differentiate between a normal query and an inner hit query (#16600) Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + .../org/opensearch/index/query/NestedQueryBuilder.java | 2 ++ .../org/opensearch/index/query/QueryShardContext.java | 9 +++++++++ .../opensearch/index/query/NestedQueryBuilderTests.java | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3086096cb8f2..8d552b5eed83b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files)) - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) +- Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) diff --git a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java index 5908882472ce7..ec7e62035a82f 100644 --- a/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/NestedQueryBuilder.java @@ -413,6 +413,7 @@ protected void doBuild(SearchContext parentSearchContext, InnerHitsContext inner try { queryShardContext.setParentFilter(parentFilter); queryShardContext.nestedScope().nextLevel(nestedObjectMapper); + queryShardContext.setInnerHitQuery(true); try { NestedInnerHitSubContext nestedInnerHits = new NestedInnerHitSubContext( name, @@ -427,6 +428,7 @@ protected void doBuild(SearchContext parentSearchContext, InnerHitsContext inner } } finally { queryShardContext.setParentFilter(previousParentFilter); + queryShardContext.setInnerHitQuery(false); } } } diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index bccead2b029d0..d717f10b17d9c 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -126,6 +126,7 @@ public class QueryShardContext extends QueryRewriteContext { private BitSetProducer parentFilter; private DerivedFieldResolver derivedFieldResolver; private boolean keywordIndexOrDocValuesEnabled; + private boolean isInnerHitQuery; public QueryShardContext( int shardId, @@ -727,4 +728,12 @@ public BitSetProducer getParentFilter() { public void setParentFilter(BitSetProducer parentFilter) { this.parentFilter = parentFilter; } + + public boolean isInnerHitQuery() { + return isInnerHitQuery; + } + + public void setInnerHitQuery(boolean isInnerHitQuery) { + this.isInnerHitQuery = isInnerHitQuery; + } } diff --git a/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java index 351011eb1b812..c367d123402d4 100644 --- a/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/NestedQueryBuilderTests.java @@ -335,6 +335,9 @@ public void testParentFilterFromInlineLeafInnerHitsNestedQuery() throws Exceptio if (context.getParentFilter() == null) { throw new Exception("Expect parent filter to be non-null"); } + if (context.isInnerHitQuery() == false) { + throw new Exception("Expect it to be inner hit query"); + } return invoke.callRealMethod(); }); NestedQueryBuilder query = new NestedQueryBuilder("nested1", innerQueryBuilder, ScoreMode.None); @@ -345,6 +348,7 @@ public void testParentFilterFromInlineLeafInnerHitsNestedQuery() throws Exceptio assertThat(innerHitBuilders.size(), Matchers.equalTo(1)); assertTrue(innerHitBuilders.containsKey(leafInnerHits.getName())); assertNull(queryShardContext.getParentFilter()); + assertFalse(queryShardContext.isInnerHitQuery()); innerHitBuilders.get(leafInnerHits.getName()).build(searchContext, innerHitsContext); assertNull(queryShardContext.getParentFilter()); verify(innerQueryBuilder).toQuery(queryShardContext); From b9d97297f1df4ec532db5b3e90c92636668a9061 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:55:42 +0800 Subject: [PATCH 321/357] Bump com.nimbusds:nimbus-jose-jwt from 9.41.1 to 9.46 in /test/fixtures/hdfs-fixture (#16611) * Bump com.nimbusds:nimbus-jose-jwt in /test/fixtures/hdfs-fixture Bumps [com.nimbusds:nimbus-jose-jwt](https://bitbucket.org/connect2id/nimbus-jose-jwt) from 9.41.1 to 9.46. - [Changelog](https://bitbucket.org/connect2id/nimbus-jose-jwt/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/nimbus-jose-jwt/branches/compare/9.46..9.41.1) --- updated-dependencies: - dependency-name: com.nimbusds:nimbus-jose-jwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d552b5eed83b..f83a1935b8dfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) +- Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611)) - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 8a402879970d7..bfa6c65ee6ec8 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -79,7 +79,7 @@ dependencies { api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.27.1' api 'org.apache.commons:commons-configuration2:2.11.0' - api 'com.nimbusds:nimbus-jose-jwt:9.41.1' + api 'com.nimbusds:nimbus-jose-jwt:9.46' api ('org.apache.kerby:kerb-admin:2.1.0') { exclude group: "org.jboss.xnio" exclude group: "org.jline" From 7f27ddca1c3dc9f4ff6fa4216c7f28171cec3db9 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 12 Nov 2024 13:03:35 +0530 Subject: [PATCH 322/357] Complete keyword changes for star tree (#16233) --------- Signed-off-by: Bharathwaj G Signed-off-by: bharath-techie --- CHANGELOG.md | 1 + .../index/mapper/StarTreeMapperIT.java | 32 +- .../lucene/index/DocValuesWriterWrapper.java | 18 + .../SortedNumericDocValuesWriterWrapper.java | 11 +- .../SortedSetDocValuesWriterWrapper.java | 58 ++ .../Composite912DocValuesReader.java | 33 +- .../Composite912DocValuesWriter.java | 62 +- .../datacube/DateDimension.java | 9 +- .../datacube/DimensionFactory.java | 28 +- .../datacube/DimensionType.java | 8 +- .../datacube/KeywordDimension.java | 82 +++ .../datacube/NumericDimension.java | 10 +- .../datacube/ReadDimension.java | 17 +- .../datacube/startree/StarTreeField.java | 12 +- .../startree/builder/BaseStarTreeBuilder.java | 228 ++++++- .../builder/OffHeapStarTreeBuilder.java | 16 +- .../builder/OnHeapStarTreeBuilder.java | 17 +- .../startree/builder/StarTreeBuilder.java | 1 + .../builder/StarTreeDocsFileManager.java | 3 + .../startree/builder/StarTreesBuilder.java | 8 +- .../fileformats/meta/StarTreeMetadata.java | 46 +- .../meta/StarTreeMetadataWriter.java | 26 +- .../startree/index/StarTreeValues.java | 62 +- .../utils/SequentialDocValuesIterator.java | 29 + .../startree/utils/StarTreeUtils.java | 12 +- .../SortedSetStarTreeValuesIterator.java | 59 ++ .../index/mapper/KeywordFieldMapper.java | 7 + .../SortedSetDocValuesWriterWrapperTests.java | 98 +++ .../AbstractStarTreeDVFormatTests.java | 126 ++++ .../StarTreeDocValuesFormatTests.java | 143 +---- .../StarTreeKeywordDocValuesFormatTests.java | 572 ++++++++++++++++++ .../datacube/startree/StarTreeTestUtils.java | 4 + .../builder/BaseStarTreeBuilderTests.java | 2 +- .../startree/builder/BuilderTestsUtils.java | 67 +- .../builder/StarTreeBuildMetricTests.java | 21 +- .../StarTreeBuilderFlushFlowTests.java | 143 ++++- .../StarTreeBuilderMergeFlowTests.java | 249 +++++++- .../builder/StarTreeBuilderTestCase.java | 34 +- .../meta/StarTreeMetadataTests.java | 8 +- .../startree/utils/StarTreeUtilsTests.java | 3 +- .../index/mapper/ObjectMapperTests.java | 11 +- .../index/mapper/StarTreeMapperTests.java | 43 +- .../search/SearchServiceStarTreeTests.java | 4 +- .../startree/MetricAggregatorTests.java | 2 +- .../startree/StarTreeFilterTests.java | 2 +- 45 files changed, 2120 insertions(+), 307 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/index/DocValuesWriterWrapper.java create mode 100644 server/src/main/java/org/apache/lucene/index/SortedSetDocValuesWriterWrapper.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java create mode 100644 server/src/test/java/org/opensearch/index/codec/composite/SortedSetDocValuesWriterWrapperTests.java create mode 100644 server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/AbstractStarTreeDVFormatTests.java create mode 100644 server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f83a1935b8dfc..30b1d5908c1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files)) - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) +- Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) ### Dependencies diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 5840884f5422a..c91c4d7bbb63b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase { .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) .build(); - private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) { try { return jsonBuilder().startObject() .startObject("composite") @@ -68,12 +68,15 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .endObject() .startArray("ordered_dimensions") .startObject() - .field("name", getDim(invalidDim, keywordDim)) + .field("name", getDim(invalidDim, ipdim)) + .endObject() + .startObject() + .field("name", "keyword_dv") .endObject() .endArray() .startArray("metrics") .startObject() - .field("name", getDim(invalidMetric, false)) + .field("name", getMetric(invalidMetric, false)) .endObject() .endArray() .endObject() @@ -99,6 +102,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "keyword") .field("doc_values", false) .endObject() + .startObject("ip") + .field("type", "ip") + .field("doc_values", false) + .endObject() .endObject() .endObject(); } catch (IOException e) { @@ -356,10 +363,19 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo } private static String getDim(boolean hasDocValues, boolean isKeyword) { + if (hasDocValues) { + return random().nextBoolean() ? "numeric" : "keyword"; + } else if (isKeyword) { + return "ip"; + } + return "numeric_dv"; + } + + private static String getMetric(boolean hasDocValues, boolean isKeyword) { if (hasDocValues) { return "numeric"; } else if (isKeyword) { - return "keyword"; + return "ip"; } return "numeric_dv"; } @@ -398,6 +414,7 @@ public void testValidCompositeIndex() { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); } assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("keyword_dv", starTreeFieldType.getDimensions().get(2).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); @@ -665,10 +682,7 @@ public void testInvalidDimCompositeIndex() { IllegalArgumentException.class, () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(true, false, false)).get() ); - assertEquals( - "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field", - ex.getMessage() - ); + assertTrue(ex.getMessage().startsWith("Aggregations not supported for the dimension field ")); } public void testMaxDimsCompositeIndex() { @@ -734,7 +748,7 @@ public void testUnsupportedDim() { () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get() ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]", ex.getMessage() ); } diff --git a/server/src/main/java/org/apache/lucene/index/DocValuesWriterWrapper.java b/server/src/main/java/org/apache/lucene/index/DocValuesWriterWrapper.java new file mode 100644 index 0000000000000..5329bad776e43 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/DocValuesWriterWrapper.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +import org.apache.lucene.search.DocIdSetIterator; + +/** + * Base wrapper class for DocValuesWriter. + */ +public interface DocValuesWriterWrapper { + T getDocValues(); +} diff --git a/server/src/main/java/org/apache/lucene/index/SortedNumericDocValuesWriterWrapper.java b/server/src/main/java/org/apache/lucene/index/SortedNumericDocValuesWriterWrapper.java index f7759fcced284..582e4c3f87f98 100644 --- a/server/src/main/java/org/apache/lucene/index/SortedNumericDocValuesWriterWrapper.java +++ b/server/src/main/java/org/apache/lucene/index/SortedNumericDocValuesWriterWrapper.java @@ -18,9 +18,9 @@ * * @opensearch.experimental */ -public class SortedNumericDocValuesWriterWrapper { +public class SortedNumericDocValuesWriterWrapper implements DocValuesWriterWrapper { - private final SortedNumericDocValuesWriter sortedNumericDocValuesWriter; + private final SortedNumericDocValuesWriter sortedNumericDocValuesWriterDelegate; /** * Sole constructor. Constructs a new {@link SortedNumericDocValuesWriterWrapper} instance. @@ -29,7 +29,7 @@ public class SortedNumericDocValuesWriterWrapper { * @param counter a counter for tracking memory usage */ public SortedNumericDocValuesWriterWrapper(FieldInfo fieldInfo, Counter counter) { - sortedNumericDocValuesWriter = new SortedNumericDocValuesWriter(fieldInfo, counter); + sortedNumericDocValuesWriterDelegate = new SortedNumericDocValuesWriter(fieldInfo, counter); } /** @@ -39,7 +39,7 @@ public SortedNumericDocValuesWriterWrapper(FieldInfo fieldInfo, Counter counter) * @param value the value to add */ public void addValue(int docID, long value) { - sortedNumericDocValuesWriter.addValue(docID, value); + sortedNumericDocValuesWriterDelegate.addValue(docID, value); } /** @@ -47,7 +47,8 @@ public void addValue(int docID, long value) { * * @return the {@link SortedNumericDocValues} instance */ + @Override public SortedNumericDocValues getDocValues() { - return sortedNumericDocValuesWriter.getDocValues(); + return sortedNumericDocValuesWriterDelegate.getDocValues(); } } diff --git a/server/src/main/java/org/apache/lucene/index/SortedSetDocValuesWriterWrapper.java b/server/src/main/java/org/apache/lucene/index/SortedSetDocValuesWriterWrapper.java new file mode 100644 index 0000000000000..95aa242535e48 --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/SortedSetDocValuesWriterWrapper.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +import org.apache.lucene.util.ByteBlockPool; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; + +/** + * A wrapper class for writing sorted set doc values. + *

    + * This class provides a convenient way to add sorted set doc values to a field + * and retrieve the corresponding {@link SortedSetDocValues} instance. + * + * @opensearch.experimental + */ +public class SortedSetDocValuesWriterWrapper implements DocValuesWriterWrapper { + + private final SortedSetDocValuesWriter sortedSetDocValuesWriterDelegate; + + /** + * Sole constructor. Constructs a new {@link SortedSetDocValuesWriterWrapper} instance. + * + * @param fieldInfo the field information for the field being written + * @param counter a counter for tracking memory usage + * @param byteBlockPool a byte block pool for allocating byte blocks + * @see SortedSetDocValuesWriter + */ + public SortedSetDocValuesWriterWrapper(FieldInfo fieldInfo, Counter counter, ByteBlockPool byteBlockPool) { + sortedSetDocValuesWriterDelegate = new SortedSetDocValuesWriter(fieldInfo, counter, byteBlockPool); + } + + /** + * Adds a bytes ref value to the sorted set doc values for the specified document. + * + * @param docID the document ID + * @param value the value to add + */ + public void addValue(int docID, BytesRef value) { + sortedSetDocValuesWriterDelegate.addValue(docID, value); + } + + /** + * Returns the {@link SortedSetDocValues} instance containing the sorted numeric doc values + * + * @return the {@link SortedSetDocValues} instance + */ + @Override + public SortedSetDocValues getDocValues() { + return sortedSetDocValuesWriterDelegate.getDocValues(); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java index 637d3250fda3f..38d3f4867e89b 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesReader.java @@ -14,7 +14,7 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; @@ -40,6 +40,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -111,7 +112,7 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState readState.segmentInfo.getId(), readState.segmentSuffix ); - + Map dimensionFieldTypeMap = new HashMap<>(); while (true) { // validate magic marker @@ -155,13 +156,16 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState compositeIndexInputMap.put(compositeFieldName, starTreeIndexInput); compositeIndexMetadataMap.put(compositeFieldName, starTreeMetadata); - List dimensionFields = starTreeMetadata.getDimensionFields(); - + Map dimensionFieldToDocValuesMap = starTreeMetadata.getDimensionFields(); // generating star tree unique fields (fully qualified name for dimension and metrics) - for (String dimensions : dimensionFields) { - fields.add(fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeFieldName, dimensions)); + for (Map.Entry dimensionEntry : dimensionFieldToDocValuesMap.entrySet()) { + String dimName = fullyQualifiedFieldNameForStarTreeDimensionsDocValues( + compositeFieldName, + dimensionEntry.getKey() + ); + fields.add(dimName); + dimensionFieldTypeMap.put(dimName, dimensionEntry.getValue()); } - // adding metric fields for (Metric metric : starTreeMetadata.getMetrics()) { for (MetricStat metricStat : metric.getBaseMetrics()) { @@ -184,7 +188,7 @@ public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState // populates the dummy list of field infos to fetch doc id set iterators for respective fields. // the dummy field info is used to fetch the doc id set iterators for respective fields based on field name - FieldInfos fieldInfos = new FieldInfos(getFieldInfoList(fields)); + FieldInfos fieldInfos = new FieldInfos(getFieldInfoList(fields, dimensionFieldTypeMap)); this.readState = new SegmentReadState( readState.directory, readState.segmentInfo, @@ -291,17 +295,4 @@ public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo comp } - /** - * Returns the sorted numeric doc values for the given sorted numeric field. - * If the sorted numeric field is null, it returns an empty doc id set iterator. - *

    - * Sorted numeric field can be null for cases where the segment doesn't hold a particular value. - * - * @param sortedNumeric the sorted numeric doc values for a field - * @return empty sorted numeric values if the field is not present, else sortedNumeric - */ - public static SortedNumericDocValues getSortedNumericDocValues(SortedNumericDocValues sortedNumeric) { - return sortedNumeric == null ? DocValues.emptySortedNumeric() : sortedNumeric; - } - } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java index dd35091dece2f..904d6a7aba5c6 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; @@ -29,12 +30,12 @@ import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.index.CompositeIndexValues; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -71,6 +72,7 @@ public class Composite912DocValuesWriter extends DocValuesConsumer { private final AtomicInteger fieldNumberAcrossCompositeFields; private final Map fieldProducerMap = new HashMap<>(); + private final Map fieldDocIdSetIteratorMap = new HashMap<>(); public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) throws IOException { @@ -82,14 +84,7 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); compositeFieldSet = new HashSet<>(); segmentFieldSet = new HashSet<>(); - // TODO : add integ test for this - for (FieldInfo fi : this.state.fieldInfos) { - if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { - segmentFieldSet.add(fi.name); - } else if (fi.name.equals(DocCountFieldMapper.NAME)) { - segmentFieldSet.add(fi.name); - } - } + addStarTreeSupportedFieldsFromSegment(); for (CompositeMappedFieldType type : compositeMappedFieldTypes) { compositeFieldSet.addAll(type.fields()); } @@ -148,6 +143,17 @@ public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false; } + private void addStarTreeSupportedFieldsFromSegment() { + // TODO : add integ test for this + for (FieldInfo fi : this.state.fieldInfos) { + if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType()) + || DocValuesType.SORTED_SET.equals(fi.getDocValuesType()) + || fi.name.equals(DocCountFieldMapper.NAME)) { + segmentFieldSet.add(fi.name); + } + } + } + @Override public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addNumericField(field, valuesProducer); @@ -179,6 +185,15 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu @Override public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addSortedSetField(field, valuesProducer); + // Perform this only during flush flow + if (mergeState.get() == null && segmentHasCompositeFields) { + createCompositeIndicesIfPossible(valuesProducer, field); + } + if (mergeState.get() != null) { + if (compositeFieldSet.contains(field.name)) { + fieldDocIdSetIteratorMap.put(field.name, valuesProducer.getSortedSet(field)); + } + } } @Override @@ -231,6 +246,7 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, * Add empty doc values for fields not present in segment */ private void addDocValuesForEmptyField(String compositeField) { + // special case for doc count if (compositeField.equals(DocCountFieldMapper.NAME)) { fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { @Override @@ -239,16 +255,31 @@ public NumericDocValues getNumeric(FieldInfo field) { } }); } else { - fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) { - return DocValues.emptySortedNumeric(); - } - }); + if (isSortedSetField(compositeField)) { + fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) { + return DocValues.emptySortedSet(); + } + }); + } + // TODO : change this logic to evaluate for sortedNumericField specifically + else { + fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return DocValues.emptySortedNumeric(); + } + }); + } } compositeFieldSet.remove(compositeField); } + private boolean isSortedSetField(String field) { + return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType; + } + @Override public void merge(MergeState mergeState) throws IOException { this.mergeState.compareAndSet(null, mergeState); @@ -272,7 +303,6 @@ private void mergeCompositeFields(MergeState mergeState) throws IOException { */ private void mergeStarTreeFields(MergeState mergeState) throws IOException { Map> starTreeSubsPerField = new HashMap<>(); - StarTreeField starTreeField = null; for (int i = 0; i < mergeState.docValuesProducers.length; i++) { CompositeIndexReader reader = null; if (mergeState.docValuesProducers[i] == null) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index 8feb9ccd27dbd..88a67e1134067 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -99,6 +99,11 @@ public List getSubDimensionNames() { return fields; } + @Override + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_NUMERIC; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("date_dimension"); @@ -170,8 +175,4 @@ public int compare(DateTimeUnitRounding unit1, DateTimeUnitRounding unit2) { public static List getSortedDateTimeUnits(List dateTimeUnits) { return dateTimeUnits.stream().sorted(new DateTimeUnitComparator()).collect(Collectors.toList()); } - - public DocValuesType getDocValuesType() { - return DocValuesType.SORTED_NUMERIC; - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 7e72a3f0d9de6..e834706e2fa9d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS; +import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD; /** * Dimension factory class mainly used to parse and create dimension from the mappings @@ -43,6 +44,8 @@ public static Dimension parseAndCreateDimension( return parseAndCreateDateDimension(name, dimensionMap, c); case NumericDimension.NUMERIC: return new NumericDimension(name); + case KEYWORD: + return new KeywordDimension(name); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name) @@ -56,16 +59,23 @@ public static Dimension parseAndCreateDimension( Map dimensionMap, Mapper.TypeParser.ParserContext c ) { - if (builder.getSupportedDataCubeDimensionType().isPresent() - && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.DATE)) { - return parseAndCreateDateDimension(name, dimensionMap, c); - } else if (builder.getSupportedDataCubeDimensionType().isPresent() - && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.NUMERIC)) { + if (builder.getSupportedDataCubeDimensionType().isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) + ); + } + switch (builder.getSupportedDataCubeDimensionType().get()) { + case DATE: + return parseAndCreateDateDimension(name, dimensionMap, c); + case NUMERIC: return new NumericDimension(name); - } - throw new IllegalArgumentException( - String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) - ); + case KEYWORD: + return new KeywordDimension(name); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) + ); + } } private static DateDimension parseAndCreateDateDimension( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java index 4b9faea331752..d327f8ca1fa1e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java @@ -27,5 +27,11 @@ public enum DimensionType { * Represents a date dimension type. * This is used for dimensions that contain date or timestamp values. */ - DATE + DATE, + + /** + * Represents a keyword dimension type. + * This is used for dimensions that contain keyword ordinals. + */ + KEYWORD } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java new file mode 100644 index 0000000000000..58e248fd548d6 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.apache.lucene.index.DocValuesType; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Composite index keyword dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class KeywordDimension implements Dimension { + public static final String KEYWORD = "keyword"; + private final String field; + + public KeywordDimension(String field) { + this.field = field; + } + + @Override + public String getField() { + return field; + } + + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public void setDimensionValues(Long value, Consumer dimSetter) { + // This will set the keyword dimension value's ordinal + dimSetter.accept(value); + } + + @Override + public List getSubDimensionNames() { + return List.of(field); + } + + @Override + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_SET; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CompositeDataCubeFieldType.NAME, field); + builder.field(CompositeDataCubeFieldType.TYPE, KEYWORD); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeywordDimension dimension = (KeywordDimension) o; + return Objects.equals(field, dimension.getField()); + } + + @Override + public int hashCode() { + return Objects.hash(field); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index f1d1b15337f4a..fe9e3d17c0047 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -51,6 +51,11 @@ public List getSubDimensionNames() { return List.of(field); } + @Override + public DocValuesType getDocValuesType() { + return DocValuesType.SORTED_NUMERIC; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -72,9 +77,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } - - @Override - public DocValuesType getDocValuesType() { - return DocValuesType.SORTED_NUMERIC; - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 0e2ec086abc0a..384553a8f7e06 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -25,9 +25,16 @@ public class ReadDimension implements Dimension { public static final String READ = "read"; private final String field; + private final DocValuesType docValuesType; public ReadDimension(String field) { this.field = field; + this.docValuesType = DocValuesType.SORTED_NUMERIC; + } + + public ReadDimension(String field, DocValuesType docValuesType) { + this.field = field; + this.docValuesType = docValuesType; } public String getField() { @@ -49,6 +56,11 @@ public List getSubDimensionNames() { return List.of(field); } + @Override + public DocValuesType getDocValuesType() { + return docValuesType; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -70,9 +82,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(field); } - - @Override - public DocValuesType getDocValuesType() { - return DocValuesType.SORTED_NUMERIC; - } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java index 833bf63c04a18..37b59fc1f59c8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree; +import org.apache.lucene.index.DocValuesType; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -33,6 +34,7 @@ public class StarTreeField implements ToXContent { private final List metrics; private final StarTreeFieldConfiguration starTreeConfig; private final List dimensionNames; + private final List dimensionDocValueTypes; private final List metricNames; public StarTreeField(String name, List dimensions, List metrics, StarTreeFieldConfiguration starTreeConfig) { @@ -41,8 +43,12 @@ public StarTreeField(String name, List dimensions, List metri this.metrics = metrics; this.starTreeConfig = starTreeConfig; dimensionNames = new ArrayList<>(); + dimensionDocValueTypes = new ArrayList<>(); for (Dimension dimension : dimensions) { - dimensionNames.addAll(dimension.getSubDimensionNames()); + for (String dimensionName : dimension.getSubDimensionNames()) { + dimensionNames.add(dimensionName); + dimensionDocValueTypes.add(dimension.getDocValuesType()); + } } metricNames = new ArrayList<>(); for (Metric metric : metrics) { @@ -64,6 +70,10 @@ public List getDimensionNames() { return dimensionNames; } + public List getDimensionDocValueTypes() { + return dimensionDocValueTypes; + } + public List getMetricNames() { return metricNames; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 3054e8e66b601..cf36f2d7d4126 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -13,15 +13,23 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.DocValuesWriterWrapper; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedNumericDocValuesWriterWrapper; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.SortedSetDocValuesWriterWrapper; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.LongValues; import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.packed.PackedInts; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -36,6 +44,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -45,6 +54,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -86,12 +97,20 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected final int maxLeafDocuments; List dimensionsSplitOrder = new ArrayList<>(); protected final InMemoryTreeNode rootNode = getNewNode(); - protected final StarTreeField starTreeField; private final SegmentWriteState writeState; private final IndexOutput metaOut; private final IndexOutput dataOut; + private final Counter bytesUsed = Counter.newCounter(); + private Map flushSortedSetDocValuesMap = new HashMap<>(); + // Maintains list of sortedSetDocValues for each star tree dimension field across segments during merge + private Map> mergeSortedSetDimensionsMap = new HashMap<>(); + // Maintains ordinalMap for each star tree dimension field during merge + private Map mergeSortedSetDimensionsOrdinalMap = new HashMap<>(); + + // This should be true for merge flows + protected boolean isMerge = false; /** * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. @@ -233,11 +252,23 @@ public void build( String dimension = dimensionsSplitOrder.get(i).getField(); FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); if (dimensionFieldInfo == null) { - dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); + dimensionFieldInfo = getFieldInfo(dimension, dimensionsSplitOrder.get(i).getDocValuesType()); } - dimensionReaders[i] = new SequentialDocValuesIterator( - new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)) + dimensionReaders[i] = getSequentialDocValuesIterator( + dimensionFieldInfo, + fieldProducerMap, + dimensionsSplitOrder.get(i).getDocValuesType() ); + + if (dimensionsSplitOrder.get(i).getDocValuesType().equals(DocValuesType.SORTED_SET)) { + // This is needed as we need to write the ordinals and also the bytesRef associated with it + // as part of star tree doc values file formats + flushSortedSetDocValuesMap.put( + dimensionsSplitOrder.get(i).getField(), + fieldProducerMap.get(dimensionFieldInfo.name).getSortedSet(dimensionFieldInfo) + ); + } + } Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -245,6 +276,72 @@ public void build( logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); } + /** + * Returns the sequential doc values iterator for the given field based on associated docValuesType + */ + private SequentialDocValuesIterator getSequentialDocValuesIterator( + FieldInfo fieldInfo, + Map fieldProducerMap, + DocValuesType type + ) throws IOException { + switch (type) { + case SORTED_NUMERIC: + return new SequentialDocValuesIterator( + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(fieldInfo.name).getSortedNumeric(fieldInfo)) + ); + case SORTED_SET: + return new SequentialDocValuesIterator( + new SortedSetStarTreeValuesIterator(fieldProducerMap.get(fieldInfo.name).getSortedSet(fieldInfo)) + ); + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + /** + * Returns the ordinal map per field based on given star-tree values across different segments + */ + protected Map getOrdinalMaps(List starTreeValuesSubs) throws IOException { + long curr = System.currentTimeMillis(); + Map> dimensionToIterators = new HashMap<>(); + // Group iterators by dimension + for (StarTreeValues starTree : starTreeValuesSubs) { + for (String dimName : starTree.getStarTreeField().getDimensionNames()) { + if (starTree.getDimensionValuesIterator(dimName) instanceof SortedSetStarTreeValuesIterator) { + dimensionToIterators.computeIfAbsent(dimName, k -> new ArrayList<>()) + .add((SortedSetStarTreeValuesIterator) starTree.getDimensionValuesIterator(dimName)); + } + } + } + + if (dimensionToIterators.isEmpty()) return Collections.emptyMap(); + this.mergeSortedSetDimensionsMap = dimensionToIterators; + Map dimensionToOrdinalMap = new HashMap<>(); + for (Map.Entry> entry : dimensionToIterators.entrySet()) { + String dimName = entry.getKey(); + List iterators = entry.getValue(); + + // step 1: iterate through each sub and mark terms still in use + TermsEnum[] liveTerms = new TermsEnum[iterators.size()]; + long[] weights = new long[liveTerms.length]; + + for (int sub = 0; sub < liveTerms.length; sub++) { + SortedSetStarTreeValuesIterator dv = iterators.get(sub); + liveTerms[sub] = dv.termsEnum(); + weights[sub] = dv.getValueCount(); + } + + // step 2: create ordinal map for this dimension + OrdinalMap map = OrdinalMap.build(null, liveTerms, weights, PackedInts.COMPACT); + dimensionToOrdinalMap.put(dimName, map); + + logger.debug("Ordinal map for dimension {} - Size in bytes: {}", dimName, map.ramBytesUsed()); + } + this.mergeSortedSetDimensionsOrdinalMap = dimensionToOrdinalMap; + logger.debug("Total time to build ordinal maps: {} ms", System.currentTimeMillis() - curr); + return dimensionToOrdinalMap; + } + /** * Builds the star tree using sorted and aggregated star-tree Documents * @@ -295,6 +392,9 @@ void appendDocumentsToStarTree(Iterator starTreeDocumentIterat } } + /** + * Writes star tree structure to file format + */ private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeDocs) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); @@ -314,10 +414,13 @@ private void serializeStarTree(int numSegmentStarTreeDocuments, int numStarTreeD ); } + /** + * Creates the star-tree docValues indices in disk + */ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, AtomicInteger fieldNumberAcrossStarTrees) throws IOException { - List dimensionWriters = new ArrayList<>(); - List metricWriters = new ArrayList<>(); + List> dimensionWriters = new ArrayList<>(); + List> metricWriters = new ArrayList<>(); FieldInfo[] dimensionFieldInfoList = new FieldInfo[numDimensions]; FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; int dimIndex = 0; @@ -325,16 +428,21 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A for (String name : dim.getSubDimensionNames()) { final FieldInfo fi = getFieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), name), - DocValuesType.SORTED_NUMERIC, + dim.getDocValuesType(), fieldNumberAcrossStarTrees.getAndIncrement() ); dimensionFieldInfoList[dimIndex] = fi; - dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + if (dim.getDocValuesType().equals(DocValuesType.SORTED_SET)) { + ByteBlockPool.DirectTrackingAllocator byteBlockAllocator = new ByteBlockPool.DirectTrackingAllocator(bytesUsed); + ByteBlockPool docValuesBytePool = new ByteBlockPool(byteBlockAllocator); + dimensionWriters.add(new SortedSetDocValuesWriterWrapper(fi, bytesUsed, docValuesBytePool)); + } else { + dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, bytesUsed)); + } dimIndex++; } } for (int i = 0; i < metricAggregatorInfos.size(); i++) { - final FieldInfo fi = getFieldInfo( fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeField.getName(), @@ -344,16 +452,18 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A DocValuesType.SORTED_NUMERIC, fieldNumberAcrossStarTrees.getAndIncrement() ); - metricFieldInfoList[i] = fi; - metricWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + metricWriters.add(new SortedNumericDocValuesWriterWrapper(fi, bytesUsed)); } - for (int docId = 0; docId < numStarTreeDocs; docId++) { StarTreeDocument starTreeDocument = getStarTreeDocument(docId); - for (int i = 0; i < starTreeDocument.dimensions.length; i++) { - if (starTreeDocument.dimensions[i] != null) { - dimensionWriters.get(i).addValue(docId, starTreeDocument.dimensions[i]); + int idx = 0; + for (Dimension dim : dimensionsSplitOrder) { + for (String name : dim.getSubDimensionNames()) { + if (starTreeDocument.dimensions[idx] != null) { + indexDocValue(dimensionWriters.get(idx), docId, starTreeDocument.dimensions[idx], dim.getField()); + } + idx++; } } @@ -362,11 +472,17 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A FieldValueConverter aggregatedValueType = metricAggregatorInfos.get(i).getValueAggregators().getAggregatedValueType(); if (aggregatedValueType.equals(LONG)) { if (starTreeDocument.metrics[i] != null) { - metricWriters.get(i).addValue(docId, (long) starTreeDocument.metrics[i]); + ((SortedNumericDocValuesWriterWrapper) (metricWriters.get(i))).addValue( + docId, + (long) starTreeDocument.metrics[i] + ); } } else if (aggregatedValueType.equals(DOUBLE)) { if (starTreeDocument.metrics[i] != null) { - metricWriters.get(i).addValue(docId, NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i])); + ((SortedNumericDocValuesWriterWrapper) (metricWriters.get(i))).addValue( + docId, + NumericUtils.doubleToSortableLong((Double) starTreeDocument.metrics[i]) + ); } } else { throw new IllegalStateException("Unknown metric doc value type"); @@ -376,26 +492,68 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A } } } - addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, numDimensions); addStarTreeDocValueFields(docValuesConsumer, metricWriters, metricFieldInfoList, metricAggregatorInfos.size()); } + /** + * Adds startree field to respective field writers + */ + private void indexDocValue(DocValuesWriterWrapper dvWriter, int docId, long value, String field) throws IOException { + if (dvWriter instanceof SortedSetDocValuesWriterWrapper) { + // TODO : cache lookupOrd to make it faster + if (isMerge) { + OrdinalMap map = mergeSortedSetDimensionsOrdinalMap.get(field); + int segmentNumber = map.getFirstSegmentNumber(value); + long segmentOrd = map.getFirstSegmentOrd(value); + ((SortedSetDocValuesWriterWrapper) dvWriter).addValue( + docId, + mergeSortedSetDimensionsMap.get(field).get(segmentNumber).lookupOrd(segmentOrd) + ); + } else { + ((SortedSetDocValuesWriterWrapper) dvWriter).addValue(docId, flushSortedSetDocValuesMap.get(field).lookupOrd(value)); + } + } else if (dvWriter instanceof SortedNumericDocValuesWriterWrapper) { + ((SortedNumericDocValuesWriterWrapper) dvWriter).addValue(docId, value); + } + } + + @SuppressWarnings("unchecked") private void addStarTreeDocValueFields( DocValuesConsumer docValuesConsumer, - List docValuesWriters, + List> docValuesWriters, FieldInfo[] fieldInfoList, int fieldCount ) throws IOException { for (int i = 0; i < fieldCount; i++) { final int writerIndex = i; - DocValuesProducer docValuesProducer = new EmptyDocValuesProducer() { - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) { - return docValuesWriters.get(writerIndex).getDocValues(); - } - }; - docValuesConsumer.addSortedNumericField(fieldInfoList[i], docValuesProducer); + DocValuesProducer docValuesProducer; + switch (fieldInfoList[i].getDocValuesType()) { + case SORTED_NUMERIC: + docValuesProducer = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + DocValuesWriterWrapper wrapper = (DocValuesWriterWrapper< + SortedNumericDocValues>) docValuesWriters.get(writerIndex); + return wrapper.getDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(fieldInfoList[i], docValuesProducer); + break; + case SORTED_SET: + docValuesProducer = new EmptyDocValuesProducer() { + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) { + DocValuesWriterWrapper wrapper = (DocValuesWriterWrapper< + SortedSetDocValues>) docValuesWriters.get(writerIndex); + return wrapper.getDocValues(); + } + }; + docValuesConsumer.addSortedSetField(fieldInfoList[i], docValuesProducer); + break; + default: + throw new IllegalStateException("Unsupported doc values type"); + } } } @@ -405,13 +563,14 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) { protected StarTreeDocument getStarTreeDocument( int currentDocId, SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + List metricReaders, + Map longValues ) throws IOException { Long[] dims = new Long[numDimensions]; int i = 0; for (SequentialDocValuesIterator dimensionValueIterator : dimensionReaders) { dimensionValueIterator.nextEntry(currentDocId); - Long val = dimensionValueIterator.value(currentDocId); + Long val = dimensionValueIterator.value(currentDocId, longValues.get(starTreeField.getDimensionNames().get(i))); dims[i] = val; i++; } @@ -431,7 +590,7 @@ protected StarTreeDocument getStarTreeDocument( /** * Sets dimensions / metric readers nnd numSegmentDocs */ - protected void setReadersAndNumSegmentDocs( + protected void setReadersAndNumSegmentDocsDuringMerge( SequentialDocValuesIterator[] dimensionReaders, List metricReaders, AtomicInteger numSegmentDocs, @@ -452,7 +611,6 @@ protected void setReadersAndNumSegmentDocs( metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } - numSegmentDocs.set( Integer.parseInt(starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS))) ); @@ -669,6 +827,14 @@ private static Long getLong(Object metric) { return metricValue; } + /** + * Sets the sortedSetDocValuesMap. + * This is needed as we need to write the ordinals and also the bytesRef associated with it + */ + void setFlushSortedSetDocValuesMap(Map flushSortedSetDocValuesMap) { + this.flushSortedSetDocValuesMap = flushSortedSetDocValuesMap; + } + /** * Merges a star-tree document into an aggregated star-tree document. * A new aggregated star-tree document is created if the aggregated document is null. @@ -799,7 +965,6 @@ private void constructStarTree(InMemoryTreeNode node, int startDocId, int endDoc constructStarTree(child, child.getStartDocId(), child.getEndDocId()); } } - } /** @@ -837,7 +1002,6 @@ private void addChildNode(InMemoryTreeNode node, int endDocId, int dimensionId, childNodeDimensionValue = nodeDimensionValue; childNodeType = StarTreeNodeType.DEFAULT.getValue(); } - InMemoryTreeNode lastNode = getNewNode(dimensionId, nodeStartDocId, endDocId, childNodeType, childNodeDimensionValue); node.addChildNode(lastNode, nodeDimensionValue); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 09d92e3da29c3..63659ef684744 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -11,8 +11,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; @@ -28,7 +30,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -177,19 +181,27 @@ private Object[] getStarTreeMetricFieldValuesFromSegment(int currentDocId, List< Iterator mergeStarTrees(List starTreeValuesSubs) throws IOException { int numDocs = 0; int[] docIds; + this.isMerge = true; + Map ordinalMaps = getOrdinalMaps(starTreeValuesSubs); try { + int seg = 0; for (StarTreeValues starTreeValues : starTreeValuesSubs) { SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); AtomicInteger numSegmentDocs = new AtomicInteger(); - setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); + setReadersAndNumSegmentDocsDuringMerge(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; + Map longValuesMap = new LinkedHashMap<>(); + for (Map.Entry entry : ordinalMaps.entrySet()) { + longValuesMap.put(entry.getKey(), entry.getValue().getGlobalOrds(seg)); + } while (currentDocId < numSegmentDocs.get()) { - StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); + StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, longValuesMap); segmentDocumentFileManager.writeStarTreeDocument(starTreeDocument, true); numDocs++; currentDocId++; } + seg++; } docIds = new int[numDocs]; for (int i = 0; i < numDocs; i++) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 07142fc5c8be7..c91f4c5db98bb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -8,8 +8,10 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -21,7 +23,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -113,6 +117,7 @@ public void build( */ @Override Iterator mergeStarTrees(List starTreeValuesSubs) throws IOException { + this.isMerge = true; return sortAndAggregateStarTreeDocuments(getSegmentsStarTreeDocuments(starTreeValuesSubs), true); } @@ -125,17 +130,23 @@ Iterator mergeStarTrees(List starTreeValuesSub */ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeValuesSubs) throws IOException { List starTreeDocuments = new ArrayList<>(); + Map ordinalMaps = getOrdinalMaps(starTreeValuesSubs); + int seg = 0; for (StarTreeValues starTreeValues : starTreeValuesSubs) { - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); AtomicInteger numSegmentDocs = new AtomicInteger(); - setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); + setReadersAndNumSegmentDocsDuringMerge(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; + Map longValuesMap = new LinkedHashMap<>(); + for (Map.Entry entry : ordinalMaps.entrySet()) { + longValuesMap.put(entry.getKey(), entry.getValue().getGlobalOrds(seg)); + } while (currentDocId < numSegmentDocs.get()) { - starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); + starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders, longValuesMap)); currentDocId++; } + seg++; } StarTreeDocument[] starTreeDocumentsArr = new StarTreeDocument[starTreeDocuments.size()]; return starTreeDocuments.toArray(starTreeDocumentsArr); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 23415ddf29132..038164c9c842d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -47,6 +47,7 @@ void build( * @param starTreeValuesSubs contains the star tree values from multiple segments * @param fieldNumberAcrossStarTrees maintains the unique field number across the fields in the star tree * @param starTreeDocValuesConsumer consumer of star-tree doc values + * * @throws IOException when we are unable to build star-tree */ void build( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 7e920b912731d..98c3e5c6d71e6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -14,6 +14,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -45,7 +46,9 @@ *

    The set of 'star-tree.documents' files is maintained, and a tracker array is used to keep track of the start document ID for each file. * Once the number of files reaches a set threshold, the files are merged. * + * @opensearch.experimental */ +@ExperimentalApi public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implements Closeable { private static final Logger logger = LogManager.getLogger(StarTreeDocsFileManager.class); private static final String STAR_TREE_DOC_FILE_NAME = "star-tree.documents"; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index bc598c9aeab7c..3d1a780c1c7ef 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -106,10 +106,10 @@ public void close() throws IOException { /** * Merges star tree fields from multiple segments * - * @param metaOut an IndexInput for star-tree metadata - * @param dataOut an IndexInput for star-tree data - * @param starTreeValuesSubsPerField starTreeValuesSubs per field - * @param starTreeDocValuesConsumer a consumer to write star-tree doc values + * @param metaOut an IndexInput for star-tree metadata + * @param dataOut an IndexInput for star-tree data + * @param starTreeValuesSubsPerField starTreeValuesSubs per field + * @param starTreeDocValuesConsumer a consumer to write star-tree doc values */ public void buildDuringMerge( IndexOutput metaOut, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 7352c215ee390..57e47b1a5b9d9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -62,9 +63,10 @@ public class StarTreeMetadata extends CompositeIndexMetadata { private final String starTreeFieldType; /** - * List of dimension fields used in the star-tree. + * Map of dimension fields to their associated DocValuesType.Insertion order needs to be maintained + * as it dictates dimensionSplitOrder */ - private final List dimensionFields; + LinkedHashMap dimensionFieldsToDocValuesMap; /** * List of metrics, containing field names and associated metric statistics. @@ -128,7 +130,7 @@ public StarTreeMetadata( this.starTreeFieldType = this.getCompositeFieldType().getName(); this.version = version; this.numberOfNodes = readNumberOfNodes(); - this.dimensionFields = readStarTreeDimensions(); + this.dimensionFieldsToDocValuesMap = readStarTreeDimensions(); this.metrics = readMetricEntries(); this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); this.starTreeDocCount = readStarTreeDocCount(); @@ -151,7 +153,7 @@ public StarTreeMetadata( * @param compositeFieldName name of the composite field. Here, name of the star-tree field. * @param compositeFieldType type of the composite field. Here, STAR_TREE field. * @param version The version of the star tree stored in the segments. - * @param dimensionFields list of dimension fields + * @param dimensionFieldsToDocValuesMap map of dimensionFields to docValues * @param metrics list of metric entries * @param segmentAggregatedDocCount segment aggregated doc count * @param starTreeDocCount the total number of star tree documents for the segment @@ -167,7 +169,7 @@ public StarTreeMetadata( IndexInput meta, Integer version, Integer numberOfNodes, - List dimensionFields, + LinkedHashMap dimensionFieldsToDocValuesMap, List metrics, Integer segmentAggregatedDocCount, Integer starTreeDocCount, @@ -183,7 +185,7 @@ public StarTreeMetadata( this.starTreeFieldType = compositeFieldType.getName(); this.version = version; this.numberOfNodes = numberOfNodes; - this.dimensionFields = dimensionFields; + this.dimensionFieldsToDocValuesMap = dimensionFieldsToDocValuesMap; this.metrics = metrics; this.segmentAggregatedDocCount = segmentAggregatedDocCount; this.starTreeDocCount = starTreeDocCount; @@ -202,15 +204,14 @@ private int readDimensionsCount() throws IOException { return meta.readVInt(); } - private List readStarTreeDimensions() throws IOException { + private LinkedHashMap readStarTreeDimensions() throws IOException { int dimensionCount = readDimensionsCount(); - List dimensionFields = new ArrayList<>(); + LinkedHashMap dimensionFieldsToDocValuesMap = new LinkedHashMap<>(); for (int i = 0; i < dimensionCount; i++) { - dimensionFields.add(meta.readString()); + dimensionFieldsToDocValuesMap.put(meta.readString(), getDocValuesType(meta, meta.readByte())); } - - return dimensionFields; + return dimensionFieldsToDocValuesMap; } private int readMetricsCount() throws IOException { @@ -314,8 +315,8 @@ public String getStarTreeFieldType() { * * @return star-tree dimension field numbers */ - public List getDimensionFields() { - return dimensionFields; + public Map getDimensionFields() { + return dimensionFieldsToDocValuesMap; } /** @@ -405,4 +406,23 @@ public int getVersion() { public int getNumberOfNodes() { return numberOfNodes; } + + private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException { + switch (b) { + case 0: + return DocValuesType.NONE; + case 1: + return DocValuesType.NUMERIC; + case 2: + return DocValuesType.BINARY; + case 3: + return DocValuesType.SORTED; + case 4: + return DocValuesType.SORTED_SET; + case 5: + return DocValuesType.SORTED_NUMERIC; + default: + throw new CorruptIndexException("invalid docvalues byte: " + b, input); + } + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 42e6f3c59866a..569692ce18893 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.store.IndexOutput; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; @@ -130,8 +131,9 @@ private static void writeMeta( metaOut.writeVInt(starTreeField.getDimensionNames().size()); // dimensions - for (String dim : starTreeField.getDimensionNames()) { - metaOut.writeString(dim); + for (int i = 0; i < starTreeField.getDimensionNames().size(); i++) { + metaOut.writeString(starTreeField.getDimensionNames().get(i)); + metaOut.writeByte(docValuesByte(starTreeField.getDimensionDocValueTypes().get(i))); } // number of metrics @@ -171,4 +173,24 @@ private static void writeMeta( metaOut.writeVLong(dataFileLength); } + + private static byte docValuesByte(DocValuesType type) { + switch (type) { + case NONE: + return 0; + case NUMERIC: + return 1; + case BINARY: + return 2; + case SORTED: + return 3; + case SORTED_SET: + return 4; + case SORTED_NUMERIC: + return 5; + default: + // BUG + throw new AssertionError("unhandled DocValuesType: " + type); + } + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 003ebeafeae45..6a13e6e789f3a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -9,9 +9,12 @@ package org.opensearch.index.compositeindex.datacube.startree.index; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -25,6 +28,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; @@ -35,7 +39,6 @@ import java.util.Set; import java.util.function.Supplier; -import static org.opensearch.index.codec.composite.composite912.Composite912DocValuesReader.getSortedNumericDocValues; import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; @@ -128,8 +131,15 @@ public StarTreeValues( // build dimensions List readDimensions = new ArrayList<>(); - for (String dimension : starTreeMetadata.getDimensionFields()) { - readDimensions.add(new ReadDimension(dimension)); + for (String dimension : starTreeMetadata.getDimensionFields().keySet()) { + readDimensions.add( + new ReadDimension( + dimension, + readState.fieldInfos.fieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeMetadata.getCompositeFieldName(), dimension) + ).getDocValuesType() + ) + ); } // star-tree field @@ -151,19 +161,25 @@ public StarTreeValues( metricValuesIteratorMap = new LinkedHashMap<>(); // get doc id set iterators for dimensions - for (String dimension : starTreeMetadata.getDimensionFields()) { + for (String dimension : starTreeMetadata.getDimensionFields().keySet()) { dimensionValuesIteratorMap.put(dimension, () -> { try { - SortedNumericDocValues dimensionSortedNumericDocValues = null; + FieldInfo dimensionfieldInfo = null; if (readState != null) { - FieldInfo dimensionfieldInfo = readState.fieldInfos.fieldInfo( + dimensionfieldInfo = readState.fieldInfos.fieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), dimension) ); - if (dimensionfieldInfo != null) { - dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); - } } - return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); + assert dimensionfieldInfo != null; + if (dimensionfieldInfo.getDocValuesType().equals(DocValuesType.SORTED_SET)) { + SortedSetDocValues dimensionSortedSetDocValues = compositeDocValuesProducer.getSortedSet(dimensionfieldInfo); + return new SortedSetStarTreeValuesIterator(getSortedSetDocValues(dimensionSortedSetDocValues)); + } else { + SortedNumericDocValues dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric( + dimensionfieldInfo + ); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); + } } catch (IOException e) { throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e); } @@ -272,4 +288,30 @@ public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetri public int getStarTreeDocumentCount() { return starTreeMetadata.getStarTreeDocCount(); } + + /** + * Returns the sorted numeric doc values for the given sorted numeric field. + * If the sorted numeric field is null, it returns an empty doc id set iterator. + *

    + * Sorted numeric field can be null for cases where the segment doesn't hold a particular value. + * + * @param sortedNumeric the sorted numeric doc values for a field + * @return empty sorted numeric values if the field is not present, else sortedNumeric + */ + static SortedNumericDocValues getSortedNumericDocValues(SortedNumericDocValues sortedNumeric) { + return sortedNumeric == null ? DocValues.emptySortedNumeric() : sortedNumeric; + } + + /** + * Returns the sortedSet doc values for the given sortedSet field. + * If the sortedSet field is null, it returns an empty doc id set iterator. + *

    + * SortedSet field can be null for cases where the segment doesn't hold a particular value. + * + * @param sortedSetDv the sortedSet doc values for a field + * @return empty sortedSet values if the field is not present, else sortedSetDv + */ + static SortedSetDocValues getSortedSetDocValues(SortedSetDocValues sortedSetDv) { + return sortedSetDv == null ? DocValues.emptySortedSet() : sortedSetDv; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 9029a451ca4d9..c4d3526648cac 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -9,8 +9,10 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; +import org.apache.lucene.util.LongValues; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; @@ -81,6 +83,33 @@ public Long value(int currentEntryId) throws IOException { } return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue(); + } else if (starTreeValuesIterator instanceof SortedSetStarTreeValuesIterator) { + if (currentEntryId < 0) { + throw new IllegalStateException("invalid entry id to fetch the next value"); + } + if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) { + throw new IllegalStateException("StarTreeValuesIterator is already exhausted"); + } + if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) { + return null; + } + return ((SortedSetStarTreeValuesIterator) starTreeValuesIterator).nextOrd(); + } else { + throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); + } + } + + public Long value(int currentEntryId, LongValues globalOrdinalLongValues) throws IOException { + if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) { + return value(currentEntryId); + } else if (starTreeValuesIterator instanceof SortedSetStarTreeValuesIterator) { + assert globalOrdinalLongValues != null; + Long val = value(currentEntryId); + // convert local ordinal to global ordinal + if (val != null) { + val = globalOrdinalLongValues.get(val); + } + return val; } else { throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java index 2aae0d4ca7e29..240a727678d6f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtils.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; /** * Util class for building star tree @@ -67,14 +68,17 @@ public static String fullyQualifiedFieldNameForStarTreeMetricsDocValues(String s * @param fields field names * @return field infos */ - public static FieldInfo[] getFieldInfoList(List fields) { + public static FieldInfo[] getFieldInfoList(List fields, Map dimDocValuesTypeMap) { FieldInfo[] fieldInfoList = new FieldInfo[fields.size()]; - // field number is not really used. We depend on unique field names to get the desired iterator int fieldNumber = 0; - for (String fieldName : fields) { - fieldInfoList[fieldNumber] = getFieldInfo(fieldName, DocValuesType.SORTED_NUMERIC, fieldNumber); + fieldInfoList[fieldNumber] = getFieldInfo( + fieldName, + // default is sortedNumeric since all metrics right now are sorted numeric + dimDocValuesTypeMap.getOrDefault(fieldName, DocValuesType.SORTED_NUMERIC), + fieldNumber + ); fieldNumber++; } return fieldInfoList; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java new file mode 100644 index 0000000000000..0cddffe5877e9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SortedSetStarTreeValuesIterator extends StarTreeValuesIterator { + + public SortedSetStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + super(docIdSetIterator); + } + + public long nextOrd() throws IOException { + return ((SortedSetDocValues) docIdSetIterator).nextOrd(); + } + + public int docValueCount() { + return ((SortedSetDocValues) docIdSetIterator).docValueCount(); + } + + public BytesRef lookupOrd(long ord) throws IOException { + return ((SortedSetDocValues) docIdSetIterator).lookupOrd(ord); + } + + public long getValueCount() { + return ((SortedSetDocValues) docIdSetIterator).getValueCount(); + } + + public long lookupTerm(BytesRef key) throws IOException { + return ((SortedSetDocValues) docIdSetIterator).lookupTerm(key); + } + + public TermsEnum termsEnum() throws IOException { + return ((SortedSetDocValues) docIdSetIterator).termsEnum(); + } + + public TermsEnum intersect(CompiledAutomaton automaton) throws IOException { + return ((SortedSetDocValues) docIdSetIterator).intersect(automaton); + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 54a1aead5fcc7..df14a5811f6a0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -59,6 +59,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -73,6 +74,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Supplier; import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -254,6 +256,11 @@ public KeywordFieldMapper build(BuilderContext context) { this ); } + + @Override + public Optional getSupportedDataCubeDimensionType() { + return Optional.of(DimensionType.KEYWORD); + } } public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers())); diff --git a/server/src/test/java/org/opensearch/index/codec/composite/SortedSetDocValuesWriterWrapperTests.java b/server/src/test/java/org/opensearch/index/codec/composite/SortedSetDocValuesWriterWrapperTests.java new file mode 100644 index 0000000000000..b0fdd712beafb --- /dev/null +++ b/server/src/test/java/org/opensearch/index/codec/composite/SortedSetDocValuesWriterWrapperTests.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.SortedSetDocValuesWriterWrapper; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.ByteBlockPool; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Counter; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Collections; + +public class SortedSetDocValuesWriterWrapperTests extends OpenSearchTestCase { + + private SortedSetDocValuesWriterWrapper wrapper; + private FieldInfo fieldInfo; + private Counter counter; + + @Override + public void setUp() throws Exception { + super.setUp(); + fieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + counter = Counter.newCounter(); + ByteBlockPool.DirectTrackingAllocator byteBlockAllocator = new ByteBlockPool.DirectTrackingAllocator(counter); + ByteBlockPool docValuesBytePool = new ByteBlockPool(byteBlockAllocator); + wrapper = new SortedSetDocValuesWriterWrapper(fieldInfo, counter, docValuesBytePool); + } + + public void testAddValue() throws IOException { + wrapper.addValue(0, new BytesRef("text1")); + wrapper.addValue(1, new BytesRef("text2")); + wrapper.addValue(2, new BytesRef("text3")); + + SortedSetDocValues docValues = wrapper.getDocValues(); + assertNotNull(docValues); + + assertEquals(0, docValues.nextDoc()); + assertEquals(0, docValues.nextOrd()); + assertEquals(1, docValues.nextDoc()); + assertEquals(1, docValues.nextOrd()); + assertEquals(2, docValues.nextDoc()); + assertEquals(2, docValues.nextOrd()); + } + + public void testGetDocValues() { + SortedSetDocValues docValues = wrapper.getDocValues(); + assertNotNull(docValues); + } + + public void testMultipleValues() throws IOException { + wrapper.addValue(0, new BytesRef("text1")); + wrapper.addValue(0, new BytesRef("text2")); + wrapper.addValue(1, new BytesRef("text3")); + + SortedSetDocValues docValues = wrapper.getDocValues(); + assertNotNull(docValues); + + assertEquals(0, docValues.nextDoc()); + assertEquals(0, docValues.nextOrd()); + assertEquals(1, docValues.nextOrd()); + assertEquals(-1, docValues.nextOrd()); + + assertEquals(1, docValues.nextDoc()); + assertEquals(2, docValues.nextOrd()); + assertEquals(-1, docValues.nextOrd()); + } +} diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/AbstractStarTreeDVFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/AbstractStarTreeDVFormatTests.java new file mode 100644 index 0000000000000..4dfd8c08575f2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/AbstractStarTreeDVFormatTests.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite912.datacube.startree; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.MapperTestUtils; +import org.opensearch.index.codec.composite.composite912.Composite912Codec; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.indices.IndicesModule; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; + +/** + * Abstract star tree doc values Lucene tests + */ +@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") +public abstract class AbstractStarTreeDVFormatTests extends BaseDocValuesFormatTestCase { + MapperService mapperService = null; + StarTreeFieldConfiguration.StarTreeBuildMode buildMode; + + public AbstractStarTreeDVFormatTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + this.buildMode = buildMode; + } + + @ParametersFactory + public static Collection parameters() { + List parameters = new ArrayList<>(); + parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP }); + parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP }); + return parameters; + } + + @BeforeClass + public static void createMapper() throws Exception { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build()); + } + + @AfterClass + public static void clearMapper() { + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + @After + public void teardown() throws IOException { + mapperService.close(); + } + + @Override + protected Codec getCodec() { + final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); + + try { + mapperService = createMapperService(getMapping()); + } catch (IOException e) { + throw new RuntimeException(e); + } + Codec codec = new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, testLogger); + return codec; + } + + public static MapperService createMapperService(XContentBuilder builder) throws IOException { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build(); + IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); + MapperService mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( + new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), + createTempDir(), + settings, + indicesModule, + "test" + ); + mapperService.merge(indexMetadata, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService; + } + + abstract XContentBuilder getMapping() throws IOException; + + public static XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); + buildFields.accept(builder); + return builder.endObject().endObject(); + } + +} diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java index f081cadc1362c..03798c6e4ce55 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeDocValuesFormatTests.java @@ -8,14 +8,9 @@ package org.opensearch.index.codec.composite912.datacube.startree; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; @@ -24,48 +19,25 @@ import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.Version; -import org.opensearch.cluster.ClusterModule; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.CheckedConsumer; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.common.unit.ByteSizeUnit; -import org.opensearch.core.common.unit.ByteSizeValue; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.MapperTestUtils; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.codec.composite.CompositeIndexReader; -import org.opensearch.index.codec.composite.composite912.Composite912Codec; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.indices.IndicesModule; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX; import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; @@ -73,48 +45,10 @@ * Star tree doc values Lucene tests */ @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") -public class StarTreeDocValuesFormatTests extends BaseDocValuesFormatTestCase { - MapperService mapperService = null; - StarTreeFieldConfiguration.StarTreeBuildMode buildMode; +public class StarTreeDocValuesFormatTests extends AbstractStarTreeDVFormatTests { public StarTreeDocValuesFormatTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { - this.buildMode = buildMode; - } - - @ParametersFactory - public static Collection parameters() { - List parameters = new ArrayList<>(); - parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP }); - parameters.add(new Object[] { StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP }); - return parameters; - } - - @BeforeClass - public static void createMapper() throws Exception { - FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build()); - } - - @AfterClass - public static void clearMapper() { - FeatureFlags.initializeFeatureFlags(Settings.EMPTY); - } - - @After - public void teardown() throws IOException { - mapperService.close(); - } - - @Override - protected Codec getCodec() { - final Logger testLogger = LogManager.getLogger(StarTreeDocValuesFormatTests.class); - - try { - mapperService = createMapperService(getExpandedMapping()); - } catch (IOException e) { - throw new RuntimeException(e); - } - Codec codec = new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, testLogger); - return codec; + super(buildMode); } public void testStarTreeDocValues() throws IOException { @@ -124,24 +58,24 @@ public void testStarTreeDocValues() throws IOException { RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); Document doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); - doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", -1)); + doc.add(new SortedNumericDocValuesField("dv1", 1)); + doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 1)); - doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", -1)); + doc.add(new SortedNumericDocValuesField("dv1", 1)); + doc.add(new SortedNumericDocValuesField("field1", -1)); iw.addDocument(doc); doc = new Document(); iw.forceMerge(1); doc.add(new SortedNumericDocValuesField("sndv", 2)); - doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", -2)); + doc.add(new SortedNumericDocValuesField("dv1", 2)); + doc.add(new SortedNumericDocValuesField("field1", -2)); iw.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("sndv", 2)); - doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", -2)); + doc.add(new SortedNumericDocValuesField("dv1", 2)); + doc.add(new SortedNumericDocValuesField("field1", -2)); iw.addDocument(doc); iw.forceMerge(1); iw.close(); @@ -217,8 +151,9 @@ public void testStarTreeDocValuesWithDeletions() throws IOException { Directory directory = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(null); conf.setMergePolicy(newLogMergePolicy()); + conf.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD); + conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE); RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); - int iterations = 3; Map map = new HashMap<>(); List allIds = new ArrayList<>(); @@ -239,17 +174,25 @@ public void testStarTreeDocValuesWithDeletions() throws IOException { doc.add(new SortedNumericDocValuesField("dv", dvValue)); map.put(sndvValue + "-" + dvValue, fieldValue + map.getOrDefault(sndvValue + "-" + dvValue, 0)); + doc.add(new NumericDocValuesField("field-ndv", fieldValue)); + iw.addDocument(doc); } iw.flush(); } iw.commit(); - // Delete random number of documents + // Update random number of documents int docsToDelete = random().nextInt(9); // Delete up to 9 documents for (int i = 0; i < docsToDelete; i++) { if (!allIds.isEmpty()) { String idToDelete = allIds.remove(random().nextInt(allIds.size() - 1)); - iw.deleteDocuments(new Term("_id", idToDelete)); + Document doc = new Document(); + doc.add(new NumericDocValuesField("field-ndv", 1L)); + iw.w.softUpdateDocuments( + new Term("_id", idToDelete), + List.of(doc), + new NumericDocValuesField(Lucene.SOFT_DELETES_FIELD, 1) + ); allIds.remove(idToDelete); } } @@ -307,6 +250,11 @@ public void testStarTreeDocValuesWithDeletions() throws IOException { directory.close(); } + @Override + protected XContentBuilder getMapping() throws IOException { + return getExpandedMapping(); + } + public static XContentBuilder getExpandedMapping() throws IOException { return topMapping(b -> { b.startObject("composite"); @@ -319,12 +267,12 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.field("name", "sndv"); b.endObject(); b.startObject(); - b.field("name", "dv"); + b.field("name", "dv1"); b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); - b.field("name", "field"); + b.field("name", "field1"); b.startArray("stats"); b.value("sum"); b.value("value_count"); @@ -351,40 +299,13 @@ public static XContentBuilder getExpandedMapping() throws IOException { b.startObject("sndv"); b.field("type", "integer"); b.endObject(); - b.startObject("dv"); + b.startObject("dv1"); b.field("type", "integer"); b.endObject(); - b.startObject("field"); + b.startObject("field1"); b.field("type", "integer"); b.endObject(); b.endObject(); }); } - - public static XContentBuilder topMapping(CheckedConsumer buildFields) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc"); - buildFields.accept(builder); - return builder.endObject().endObject(); - } - - public static MapperService createMapperService(XContentBuilder builder) throws IOException { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) - .build(); - IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build(); - IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); - MapperService mapperService = MapperTestUtils.newMapperServiceWithHelperAnalyzer( - new NamedXContentRegistry(ClusterModule.getNamedXWriteables()), - createTempDir(), - settings, - indicesModule, - "test" - ); - mapperService.merge(indexMetadata, MapperService.MergeReason.INDEX_TEMPLATE); - return mapperService; - } } diff --git a/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java new file mode 100644 index 0000000000000..402ed1dbee98a --- /dev/null +++ b/server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/StarTreeKeywordDocValuesFormatTests.java @@ -0,0 +1,572 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.composite912.datacube.startree; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; +import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.assertStarTreeDocuments; + +/** + * Star tree doc values Lucene tests + */ +@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") +public class StarTreeKeywordDocValuesFormatTests extends AbstractStarTreeDVFormatTests { + + public StarTreeKeywordDocValuesFormatTests(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { + super(buildMode); + } + + public void testStarTreeKeywordDocValues() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new StringField("_id", "1", Field.Store.NO)); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new StringField("_id", "2", Field.Store.NO)); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22"))); + iw.addDocument(doc); + iw.flush(); + iw.deleteDocuments(new Term("_id", "2")); + iw.flush(); + doc = new Document(); + doc.add(new StringField("_id", "3", Field.Store.NO)); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new StringField("_id", "4", Field.Store.NO)); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22"))); + iw.addDocument(doc); + iw.flush(); + iw.deleteDocuments(new Term("_id", "4")); + iw.flush(); + iw.forceMerge(1); + + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Star tree documents + /** + keyword1 keyword2 | [ sum, value_count, min, max[sndv]] , doc_count + [0, 0] | [3.0, 2.0, 1.0, 2.0, 2.0] + [1, 1] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, 0] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, 1] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, null] | [6.0, 4.0, 1.0, 2.0, 4.0] + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[5]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 0L, 0L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 1L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, 0L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[4] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 6.0, 4.0, 1.0, 2.0, 4.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); + directory.close(); + } + + public void testStarTreeKeywordDocValuesWithDeletions() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + + int iterations = 3; + Set allIds = new HashSet<>(); + Map documents = new HashMap<>(); + Map map = new HashMap<>(); + for (int iter = 0; iter < iterations; iter++) { + // Add 10 documents + for (int i = 0; i < 10; i++) { + String id = String.valueOf(random().nextInt() + 1); + allIds.add(id); + Document doc = new Document(); + doc.add(new StringField("_id", id, Field.Store.YES)); + int sndvValue = random().nextInt(5) + 1; + doc.add(new SortedNumericDocValuesField("sndv", sndvValue)); + + String keyword1Value = "text" + random().nextInt(3); + + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef(keyword1Value))); + String keyword2Value = "text" + random().nextInt(3); + + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef(keyword2Value))); + map.put(keyword1Value + "-" + keyword2Value, sndvValue + map.getOrDefault(keyword1Value + "-" + keyword2Value, 0)); + iw.addDocument(doc); + documents.put(id, doc); + } + + iw.flush(); + + // Update random number of documents + int docsToDelete = random().nextInt(5); // Delete up to 5 documents + for (int i = 0; i < docsToDelete; i++) { + if (!allIds.isEmpty()) { + String idToDelete = allIds.iterator().next(); + Document doc = new Document(); + doc.add(new NumericDocValuesField("field-ndv", 1L)); + iw.w.softUpdateDocuments( + new Term("_id", idToDelete), + List.of(doc), + new NumericDocValuesField(Lucene.SOFT_DELETES_FIELD, 1) + ); + allIds.remove(idToDelete); + documents.remove(idToDelete); + } + } + + iw.flush(); + } + + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Assert star tree documents + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] actualStarTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + SortedSetStarTreeValuesIterator k1 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator( + "keyword1" + ); + SortedSetStarTreeValuesIterator k2 = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator( + "keyword2" + ); + for (StarTreeDocument starDoc : actualStarTreeDocuments) { + String keyword1 = null; + if (starDoc.dimensions[0] != null) { + keyword1 = k1.lookupOrd(starDoc.dimensions[0]).utf8ToString(); + } + + String keyword2 = null; + if (starDoc.dimensions[1] != null) { + keyword2 = k2.lookupOrd(starDoc.dimensions[1]).utf8ToString(); + } + double metric = (double) starDoc.metrics[0]; + if (map.containsKey(keyword1 + "-" + keyword2)) { + assertEquals((int) map.get(keyword1 + "-" + keyword2), (int) metric); + } + } + } + } + + ir.close(); + directory.close(); + } + + public void testStarKeywordDocValuesWithMissingDocs() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22"))); + iw.addDocument(doc); + iw.forceMerge(1); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22"))); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Star tree documents + /** + * keyword1 keyword2 | [ sum, value_count, min, max[sndv]] , doc_count + [0, 0] | [2.0, 1.0, 2.0, 2.0, 1.0] + [1, 1] | [2.0, 1.0, 2.0, 2.0, 1.0] + [null, 0] | [1.0, 1.0, 1.0, 1.0, 1.0] + [null, 1] | [1.0, 1.0, 1.0, 1.0, 1.0] + [null, 0] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, 1] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, null] | [6.0, 4.0, 1.0, 2.0, 4.0] + [null, null] | [2.0, 2.0, 1.0, 1.0, 2.0] + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[8]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 0L, 0L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 1L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, 0L }, new Double[] { 1.0, 1.0, 1.0, 1.0, 1.0 }); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 1.0, 1.0, 1.0, 1.0, 1.0 }); + expectedStarTreeDocuments[4] = new StarTreeDocument(new Long[] { null, 0L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[5] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 3.0, 2.0, 1.0, 2.0, 2.0 }); + expectedStarTreeDocuments[6] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 6.0, 4.0, 1.0, 2.0, 4.0 }); + expectedStarTreeDocuments[7] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 2.0, 2.0, 1.0, 1.0, 2.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); + directory.close(); + } + + public void testStarKeywordDocValuesWithMissingDocsInSegment() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + iw.forceMerge(1); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text2"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text11"))); + doc.add(new SortedSetDocValuesField("keyword2", new BytesRef("text22"))); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Star tree documents + /** + * keyword1 keyword2 | [ sum, value_count, min, max[sndv]] , doc_count + [0, 0] | [2.0, 1.0, 2.0, 2.0, 1.0] + [1, 1] | [2.0, 1.0, 2.0, 2.0, 1.0] + [null, null] | [2.0, 2.0, 1.0, 1.0, 2.0] // This is for missing doc + [null, 0] | [2.0, 1.0, 2.0, 2.0, 1.0] + [null, 1] | [2.0, 1.0, 2.0, 2.0, 1.0] + [null, null] | [2.0, 2.0, 1.0, 1.0, 2.0] + [null, null] | [6.0, 4.0, 1.0, 2.0, 4.0] // This is star document + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[7]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 0L, 0L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 1L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 2.0, 2.0, 1.0, 1.0, 2.0 }); + expectedStarTreeDocuments[3] = new StarTreeDocument(new Long[] { null, 0L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[4] = new StarTreeDocument(new Long[] { null, 1L }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[5] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 2.0, 2.0, 1.0, 1.0, 2.0 }); + expectedStarTreeDocuments[6] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 6.0, 4.0, 1.0, 2.0, 4.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); + directory.close(); + } + + public void testStarKeywordDocValuesWithMissingDocsInAllSegments() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + iw.forceMerge(1); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Star tree documents + /** + * keyword1 keyword2 | [ sum, value_count, min, max[sndv]] , doc_count + [null, null] | [6.0, 4.0, 1.0, 2.0, 4.0] + + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[1]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 6.0, 4.0, 1.0, 2.0, 4.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); + directory.close(); + } + + public void testStarKeywordDocValuesWithMissingDocsInMixedSegments() throws IOException { + Directory directory = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(null); + conf.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 1)); + iw.addDocument(doc); + iw.forceMerge(1); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + doc.add(new SortedSetDocValuesField("keyword1", new BytesRef("text1"))); + iw.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("sndv", 2)); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + + DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(directory)); + TestUtil.checkReader(ir); + assertEquals(1, ir.leaves().size()); + + // Star tree documents + /** + * keyword1 keyword2 | [ sum, value_count, min, max[sndv]] , doc_count + [0, 0] | [2.0, 1.0, 2.0, 2.0, 1.0] + [1, 1] | [2.0, 1.0, 2.0, 2.0, 1.0] + [null, 0] | [1.0, 1.0, 1.0, 1.0, 1.0] + [null, 1] | [1.0, 1.0, 1.0, 1.0, 1.0] + [null, 0] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, 1] | [3.0, 2.0, 1.0, 2.0, 2.0] + [null, null] | [6.0, 4.0, 1.0, 2.0, 4.0] + [null, null] | [2.0, 2.0, 1.0, 1.0, 2.0] + */ + StarTreeDocument[] expectedStarTreeDocuments = new StarTreeDocument[3]; + expectedStarTreeDocuments[0] = new StarTreeDocument(new Long[] { 0L, null }, new Double[] { 2.0, 1.0, 2.0, 2.0, 1.0 }); + expectedStarTreeDocuments[1] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 4.0, 3.0, 1.0, 2.0, 3.0 }); + expectedStarTreeDocuments[2] = new StarTreeDocument(new Long[] { null, null }, new Double[] { 6.0, 4.0, 1.0, 2.0, 4.0 }); + + for (LeafReaderContext context : ir.leaves()) { + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); + + for (CompositeIndexFieldInfo compositeIndexFieldInfo : compositeIndexFields) { + StarTreeValues starTreeValues = (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(compositeIndexFieldInfo); + StarTreeDocument[] starTreeDocuments = StarTreeTestUtils.getSegmentsStarTreeDocuments( + List.of(starTreeValues), + List.of( + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.DOUBLE, + NumberFieldMapper.NumberType.LONG + ), + Integer.parseInt(starTreeValues.getAttributes().get(STAR_TREE_DOCS_COUNT)) + ); + assertStarTreeDocuments(starTreeDocuments, expectedStarTreeDocuments); + } + } + ir.close(); + directory.close(); + } + + @Override + protected XContentBuilder getMapping() throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("max_leaf_docs", 1); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); + b.startObject(); + b.field("name", "keyword2"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "sndv"); + b.startArray("stats"); + b.value("sum"); + b.value("value_count"); + b.value("avg"); + b.value("min"); + b.value("max"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("sndv"); + b.field("type", "integer"); + b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); + b.startObject("keyword2"); + b.field("type", "keyword"); + b.endObject(); + b.endObject(); + }); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index dc8b3320f3de2..44e40f1db4cc8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -99,6 +99,7 @@ public static StarTreeDocument getStarTreeDocument( ) throws IOException { Long[] dims = new Long[dimensionReaders.length]; int i = 0; + for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { dimensionDocValueIterator.nextEntry(currentDocId); Long val = dimensionDocValueIterator.value(currentDocId); @@ -117,6 +118,9 @@ public static StarTreeDocument getStarTreeDocument( public static Double toAggregatorValueType(Long value, FieldValueConverter fieldValueConverter) { try { + if (value == null) { + return 0.0; + } return fieldValueConverter.toDoubleValue(value); } catch (Exception e) { throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 012d04c575f55..ac729f6392f63 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -201,7 +201,7 @@ public List getStarTreeDocuments() { @Override public Long getDimensionValue(int docId, int dimensionId) throws IOException { - return 0l; + return 0L; } @Override diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java index cc6c1758697dd..077bf0422ab50 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BuilderTestsUtils.java @@ -18,11 +18,13 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.index.codec.composite.LuceneDocValuesProducerFactory; @@ -150,6 +152,65 @@ public long cost() { }; } + public static SortedSetDocValues getSortedSetMock(List dimList, List docsWithField) { + return getSortedSetMock(dimList, docsWithField, 1); + } + + public static SortedSetDocValues getSortedSetMock(List dimList, List docsWithField, int valueCount) { + return new SortedSetDocValues() { + int index = -1; + + @Override + public long nextOrd() throws IOException { + return dimList.get(index); + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public BytesRef lookupOrd(long l) throws IOException { + return new BytesRef("dummy" + l); + } + + @Override + public long getValueCount() { + return valueCount; + } + + @Override + public boolean advanceExact(int target) { + return false; + } + + @Override + public int docID() { + return index; + } + + @Override + public int nextDoc() { + if (index == docsWithField.size() - 1) { + return NO_MORE_DOCS; + } + index++; + return docsWithField.get(index); + } + + @Override + public int advance(int target) { + return 0; + } + + @Override + public long cost() { + return 0; + } + }; + } + public static void validateStarTree( InMemoryTreeNode root, int totalDimensions, @@ -386,7 +447,7 @@ public static void validateStarTreeFileFormats( public static SegmentReadState getReadState( int numDocs, - List dimensionFields, + Map dimensionFields, List metrics, StarTreeField compositeField, SegmentWriteState writeState, @@ -401,7 +462,7 @@ public static SegmentReadState getReadState( FieldInfo[] fields = new FieldInfo[dimensionFields.size() + numMetrics]; int i = 0; - for (String dimension : dimensionFields) { + for (String dimension : dimensionFields.keySet()) { fields[i] = new FieldInfo( fullyQualifiedFieldNameForStarTreeDimensionsDocValues(compositeField.getName(), dimension), i, @@ -409,7 +470,7 @@ public static SegmentReadState getReadState( false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.SORTED_NUMERIC, + dimensionFields.get(dimension), -1, Collections.emptyMap(), 0, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java index c4d6fe6f19812..95adae9335740 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuildMetricTests.java @@ -52,6 +52,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -591,13 +592,19 @@ public void test_build_multipleStarTrees() throws IOException { metaOut.close(); dataOut.close(); + LinkedHashMap fieldsMap = new LinkedHashMap<>(); + fieldsMap.put("field1", DocValuesType.SORTED_NUMERIC); + fieldsMap.put("field3", DocValuesType.SORTED_NUMERIC); + fieldsMap.put("field5", DocValuesType.SORTED_NUMERIC); + fieldsMap.put("field8", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "test", STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, builder.numStarTreeNodes, - List.of("field1", "field3", "field5", "field8"), + fieldsMap, List.of( new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM)), @@ -614,13 +621,18 @@ public void test_build_multipleStarTrees() throws IOException { 330 ); + LinkedHashMap fieldsMap1 = new LinkedHashMap<>(); + fieldsMap1.put("fieldC", DocValuesType.SORTED_NUMERIC); + fieldsMap1.put("fieldB", DocValuesType.SORTED_NUMERIC); + fieldsMap1.put("fieldL", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata2 = new StarTreeMetadata( "test", STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, builder.numStarTreeNodes, - List.of("fieldC", "fieldB", "fieldL"), + fieldsMap1, List.of(new Metric("fieldI", List.of(MetricStat.SUM))), 7, 27, @@ -631,9 +643,8 @@ public void test_build_multipleStarTrees() throws IOException { 1287 ); - List totalDimensionFields = new ArrayList<>(); - totalDimensionFields.addAll(starTreeMetadata.getDimensionFields()); - totalDimensionFields.addAll(starTreeMetadata2.getDimensionFields()); + LinkedHashMap totalDimensionFields = new LinkedHashMap<>(starTreeMetadata.getDimensionFields()); + totalDimensionFields.putAll(starTreeMetadata2.getDimensionFields()); List metrics = new ArrayList<>(); metrics.addAll(starTreeMetadata.getMetrics()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java index a92ac39cb7020..440268f1f803c 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderFlushFlowTests.java @@ -9,15 +9,18 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.KeywordDimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; @@ -27,17 +30,20 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedNumericMock; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedSetMock; import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; import static org.opensearch.index.mapper.CompositeMappedFieldType.CompositeFieldType.STAR_TREE; @@ -124,14 +130,16 @@ public void testFlushFlow() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", DocValuesType.SORTED_NUMERIC); + docValues.put("field3", DocValuesType.SORTED_NUMERIC); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, builder.numStarTreeNodes, - List.of("field1", "field3"), + docValues, List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), 6, builder.numStarTreeDocs, @@ -222,13 +230,16 @@ public void testFlushFlowDimsReverse() throws IOException { dataOut.close(); docValuesConsumer.close(); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", DocValuesType.SORTED_NUMERIC); + docValues.put("field3", DocValuesType.SORTED_NUMERIC); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( "sf", STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, builder.numStarTreeNodes, - List.of("field1", "field3"), + docValues, List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), 6, builder.numStarTreeDocs, @@ -322,7 +333,10 @@ public void testFlushFlowBuild() throws IOException { dataOut.close(); docValuesConsumer.close(); - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 100, 1, 6699); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 100, 1, 6699); validateStarTreeFileFormats( builder.getRootNode(), @@ -396,6 +410,115 @@ public void testFlushFlowWithTimestamps() throws IOException { validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); } + public void testFlushFlowForKeywords() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField = List.of(0, 1, 2, 3, 4, 5); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + compositeField = getStarTreeFieldWithKeywordField(); + SortedSetStarTreeValuesIterator d1sndv = new SortedSetStarTreeValuesIterator(getSortedSetMock(dimList, docsWithField)); + SortedSetStarTreeValuesIterator d2sndv = new SortedSetStarTreeValuesIterator(getSortedSetMock(dimList2, docsWithField2)); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + + writeState = getWriteState(6, writeState.segmentInfo.getId()); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [0, 0] | [0.0, 1] + [1, 1] | [10.0, 1] + [2, 2] | [20.0, 1] + [3, 3] | [30.0, 1] + [4, 4] | [40.0, 1] + [5, 5] | [50.0, 1] + */ + + SegmentWriteState w = getWriteState(DocIdSetIterator.NO_MORE_DOCS, writeState.segmentInfo.getId()); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + w, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + Map dv = new LinkedHashMap<>(); + dv.put("field1", getSortedSetMock(dimList, docsWithField)); + dv.put("field3", getSortedSetMock(dimList2, docsWithField2)); + builder.setFlushSortedSetDocValuesMap(dv); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + + List starTreeDocuments = builder.getStarTreeDocuments(); + int count = 0; + for (StarTreeDocument starTreeDocument : starTreeDocuments) { + count++; + if (starTreeDocument.dimensions[1] != null) { + assertEquals( + starTreeDocument.dimensions[0] == null + ? starTreeDocument.dimensions[1] * 1 * 10.0 + : starTreeDocument.dimensions[0] * 10, + starTreeDocument.metrics[0] + ); + assertEquals(1L, starTreeDocument.metrics[1]); + } else { + assertEquals(150D, starTreeDocument.metrics[0]); + assertEquals(6L, starTreeDocument.metrics[1]); + } + } + assertEquals(13, count); + validateStarTree(builder.getRootNode(), 2, 1000, builder.getStarTreeDocuments()); + + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + LinkedHashMap docValues = new LinkedHashMap<>(); + docValues.put("field1", DocValuesType.SORTED_SET); + docValues.put("field3", DocValuesType.SORTED_SET); + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + docValues, + List.of(new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT, MetricStat.AVG))), + 6, + builder.numStarTreeDocs, + 1000, + Set.of(), + getBuildMode(), + 0, + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + + } + private StarTreeField getStarTreeFieldWithMultipleMetrics() { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); @@ -408,6 +531,18 @@ private StarTreeField getStarTreeFieldWithMultipleMetrics() { return new StarTreeField("sf", dims, metrics, c); } + private StarTreeField getStarTreeFieldWithKeywordField() { + Dimension d1 = new KeywordDimension("field1"); + Dimension d2 = new KeywordDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m3 = new Metric("field2", List.of(MetricStat.AVG)); + List dims = List.of(d1, d2); + List metrics = List.of(m1, m2, m3); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); + return new StarTreeField("sf", dims, metrics, c); + } + private static DocValuesProducer getDocValuesProducer(SortedNumericDocValues sndv) { return new EmptyDocValuesProducer() { @Override diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java index 00e53534a7606..be16961e781db 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderMergeFlowTests.java @@ -9,8 +9,10 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.LuceneDocValuesConsumerFactory; @@ -26,6 +28,7 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; @@ -49,6 +52,7 @@ import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedNumericMock; +import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.getSortedSetMock; import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.traverseStarTree; import static org.opensearch.index.compositeindex.datacube.startree.builder.BuilderTestsUtils.validateStarTree; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; @@ -348,8 +352,10 @@ public void testMergeFlowWithSum() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( builder.getRootNode(), @@ -421,8 +427,10 @@ public void testMergeFlowWithCount() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( builder.getRootNode(), @@ -568,8 +576,10 @@ public void testMergeFlowWithMissingDocs() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( builder.getRootNode(), @@ -656,8 +666,10 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 231); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 231); validateStarTreeFileFormats( builder.getRootNode(), @@ -747,8 +759,10 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 7, 1000, 231); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 7, 1000, 231); validateStarTreeFileFormats( builder.getRootNode(), @@ -834,8 +848,10 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( builder.getRootNode(), @@ -919,8 +935,10 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 10, 1000, 363); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 10, 1000, 363); validateStarTreeFileFormats( builder.getRootNode(), @@ -992,8 +1010,10 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 6, 1000, 264); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 6, 1000, 264); validateStarTreeFileFormats( builder.getRootNode(), @@ -1391,8 +1411,10 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { metaOut.close(); dataOut.close(); docValuesConsumer.close(); - - StarTreeMetadata starTreeMetadata = getStarTreeMetadata(List.of("field1", "field3"), 9, 1000, 330); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("field1", DocValuesType.SORTED_NUMERIC); + map.put("field3", DocValuesType.SORTED_NUMERIC); + StarTreeMetadata starTreeMetadata = getStarTreeMetadata(map, 9, 1000, 330); validateStarTreeFileFormats( builder.getRootNode(), @@ -1776,6 +1798,110 @@ public void testMergeFlowWithTimestamps() throws IOException { ); } + public void testMergeFlowWithKeywords() throws IOException { + List dimList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List docsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList1 = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField1 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(0L, 1L, 2L, 3L, -1L); + List docsWithField3 = List.of(0, 1, 2, 3, 4); + List dimList4 = List.of(0L, 1L, 2L, 3L, -1L); + List docsWithField4 = List.of(0, 1, 2, 3, 4); + List metricsList21 = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0) + ); + List metricsWithField21 = List.of(0, 1, 2, 3, 4); + List metricsList2 = List.of(0L, 1L, 2L, 3L, 4L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + compositeField = getStarTreeFieldWithKeywords(); + StarTreeValues starTreeValues = getStarTreeValuesWithKeywords( + getSortedSetMock(dimList, docsWithField), + getSortedSetMock(dimList2, docsWithField2), + getSortedNumericMock(metricsList, metricsWithField), + getSortedNumericMock(metricsList1, metricsWithField1), + compositeField, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValuesWithKeywords( + getSortedSetMock(dimList3, docsWithField3), + getSortedSetMock(dimList4, docsWithField4), + getSortedNumericMock(metricsList2, metricsWithField2), + getSortedNumericMock(metricsList21, metricsWithField21), + compositeField, + "4" + ); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite912DocValuesFormat.META_DOC_VALUES_CODEC, + Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); + // Initialize the mock MergeState within the method + + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [0, 0] | [5, 50.0] + [1, 1] | [7, 70.0] + [2, 2] | [9, 90.0] + [3, 3] | [11, 110.0] + [4, 4] | [4, 40.0] + [5, 5] | [5, 50.0] + */ + int count = 0; + builder.appendDocumentsToStarTree(starTreeDocumentIterator); + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + count++; + if (count <= 4) { + assertEquals(starTreeDocument.dimensions[0] * 2, (long) starTreeDocument.metrics[0], 0); + assertEquals(starTreeDocument.dimensions[0] * 20.0, (double) starTreeDocument.metrics[1], 0); + } else { + assertEquals(starTreeDocument.dimensions[0], (long) starTreeDocument.metrics[0], 0); + assertEquals(starTreeDocument.dimensions[0] * 10.0, (double) starTreeDocument.metrics[1], 0); + } + } + assertEquals(6, count); + builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); + validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = getStarTreeMetadata( + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + 6, + compositeField.getStarTreeConfig().maxLeafDocs(), + 264 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); + } + private StarTreeValues getStarTreeValuesWithDates( SortedNumericDocValues dimList, SortedNumericDocValues dimList2, @@ -1857,6 +1983,93 @@ private StarTreeValues getStarTreeValues( return starTreeValues; } + private StarTreeValues getStarTreeValuesWithKeywords( + SortedSetDocValues dimList, + SortedSetDocValues dimList2, + SortedNumericDocValues metricsList, + SortedNumericDocValues metricsList1, + StarTreeField sf, + String number + ) { + SortedSetDocValues d1sndv = dimList; + SortedSetDocValues d2sndv = dimList2; + SortedNumericDocValues m1sndv = metricsList; + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedSetStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedSetStarTreeValuesIterator(d2sndv) + ); + + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList) + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(1).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList1) + ); + StarTreeValues starTreeValues = new StarTreeValues( + sf, + null, + dimDocIdSetIterators, + metricDocIdSetIterators, + Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number), + null + ); + return starTreeValues; + } + + private StarTreeValues getStarTreeValuesWithKeywords( + SortedSetDocValues dimList, + SortedSetDocValues dimList2, + SortedSetDocValues dimList4, + SortedSetDocValues dimList3, + SortedNumericDocValues metricsList, + SortedNumericDocValues metricsList1, + StarTreeField sf, + String number + ) { + Map> dimDocIdSetIterators = Map.of( + "field1_minute", + () -> new SortedSetStarTreeValuesIterator(dimList), + "field1_half-hour", + () -> new SortedSetStarTreeValuesIterator(dimList4), + "field1_hour", + () -> new SortedSetStarTreeValuesIterator(dimList2), + "field3", + () -> new SortedSetStarTreeValuesIterator(dimList3) + ); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList) + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(1).getTypeName() + ), + () -> new SortedNumericStarTreeValuesIterator(metricsList1) + ); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number), null); + } + private StarTreeValues getStarTreeValues( List dimList1, List docsWithField1, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java index 6733cac12f657..9c9beaea4f52c 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; @@ -31,6 +32,7 @@ import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit; import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.KeywordDimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; @@ -60,6 +62,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -87,6 +90,7 @@ public abstract class StarTreeBuilderTestCase extends OpenSearchTestCase { protected String dataFileName; protected String metaFileName; protected List dimensionsOrder; + protected MergeState mergeState; public StarTreeBuilderTestCase(StarTreeFieldConfiguration.StarTreeBuildMode buildMode) { this.buildMode = buildMode; @@ -155,6 +159,8 @@ public void setup() throws IOException { } writeState = getWriteState(5, UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8)); + mergeState = new MergeState(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false); + dataFileName = IndexFileNames.segmentFileName( writeState.segmentInfo.name, writeState.segmentSuffix, @@ -240,7 +246,7 @@ SegmentWriteState getWriteState(int numDocs, byte[] id) { return BuilderTestsUtils.getWriteState(numDocs, id, fieldsInfo, directory); } - SegmentReadState getReadState(int numDocs, List dimensionFields, List metrics) { + SegmentReadState getReadState(int numDocs, Map dimensionFields, List metrics) { return BuilderTestsUtils.getReadState(numDocs, dimensionFields, metrics, compositeField, writeState, directory); } @@ -248,10 +254,12 @@ protected Map getAttributes(int numSegmentDocs) { return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); } - protected List getStarTreeDimensionNames(List dimensionsOrder) { - List dimensionNames = new ArrayList<>(); + protected LinkedHashMap getStarTreeDimensionNames(List dimensionsOrder) { + LinkedHashMap dimensionNames = new LinkedHashMap<>(); for (Dimension dimension : dimensionsOrder) { - dimensionNames.addAll(dimension.getSubDimensionNames()); + for (String dimensionName : dimension.getSubDimensionNames()) { + dimensionNames.put(dimensionName, dimension.getDocValuesType()); + } } return dimensionNames; } @@ -320,7 +328,12 @@ protected long getLongFromDouble(double value) { return NumericUtils.doubleToSortableLong(value); } - protected StarTreeMetadata getStarTreeMetadata(List fields, int segmentAggregatedDocCount, int maxLeafDocs, int dataLength) { + protected StarTreeMetadata getStarTreeMetadata( + LinkedHashMap fields, + int segmentAggregatedDocCount, + int maxLeafDocs, + int dataLength + ) { return new StarTreeMetadata( "sf", STAR_TREE, @@ -339,6 +352,17 @@ protected StarTreeMetadata getStarTreeMetadata(List fields, int segmentA ); } + protected StarTreeField getStarTreeFieldWithKeywords() { + Dimension d1 = new KeywordDimension("field1"); + Dimension d2 = new KeywordDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.VALUE_COUNT, MetricStat.SUM)); + List dims = List.of(d1, d2); + List metrics = List.of(m1); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(10, new HashSet<>(), getBuildMode()); + StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + return sf; + } + protected StarTreeField getStarTreeFieldWithDateDimension() { List intervals = new ArrayList<>(); intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index c8636426449ad..cc91d69be97c1 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -181,8 +182,11 @@ public void test_starTreeMetadata() throws IOException { assertEquals(starTreeMetadata.getNumberOfNodes(), numberOfNodes); assertNotNull(starTreeMetadata); - for (int i = 0; i < dimensionsOrder.size(); i++) { - assertEquals(dimensionsOrder.get(i).getField(), starTreeMetadata.getDimensionFields().get(i)); + assertEquals(dimensionsOrder.size(), starTreeMetadata.dimensionFieldsToDocValuesMap.size()); + int k = 0; + for (Map.Entry entry : starTreeMetadata.dimensionFieldsToDocValuesMap.entrySet()) { + assertEquals(dimensionsOrder.get(k).getField(), entry.getKey()); + k++; } assertEquals(starTreeField.getMetrics().size(), starTreeMetadata.getMetrics().size()); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java index 9cca0b04e9ea4..7e438c18d9ab9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeUtilsTests.java @@ -17,6 +17,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.UUID; @@ -43,7 +44,7 @@ public void testFullyQualifiedFieldNameForStarTreeMetricsDocValues() { public void testGetFieldInfoList() { List fieldNames = Arrays.asList("field1", "field2", "field3"); - FieldInfo[] actualFieldInfos = StarTreeUtils.getFieldInfoList(fieldNames); + FieldInfo[] actualFieldInfos = StarTreeUtils.getFieldInfoList(fieldNames, new HashMap<>()); for (int i = 0; i < fieldNames.size(); i++) { assertFieldInfos(actualFieldInfos[i], fieldNames.get(i), i); } diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index c35cf3fc1e591..77534b514a59a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -540,9 +540,14 @@ public void testCompositeFields() throws Exception { .endObject() .toString(); + Settings settings = Settings.builder() + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + .build(); + IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> createIndex("invalid").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)) + () -> createIndex("invalid", settings).mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)) ); assertEquals( "star tree index is under an experimental feature and can be activated only by enabling opensearch.experimental.feature.composite_index.star_tree.enabled feature flag in the JVM options", @@ -552,10 +557,6 @@ public void testCompositeFields() throws Exception { final Settings starTreeEnabledSettings = Settings.builder().put(STAR_TREE_INDEX, "true").build(); FeatureFlags.initializeFeatureFlags(starTreeEnabledSettings); - Settings settings = Settings.builder() - .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) - .build(); DocumentMapper documentMapper = createIndex("test", settings).mapperService() .documentMapperParser() .parse("tweet", new CompressedXContent(mapping)); diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index aac460bd5e332..8ec34b3eb660c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -672,6 +672,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -718,6 +721,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() + .endObject() .endObject(); } catch (IOException e) { @@ -772,6 +776,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -823,6 +830,9 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -866,6 +876,9 @@ private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExcee b.startObject(); b.field("name", "metric_field"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); @@ -895,6 +908,9 @@ private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExcee b.startObject("metric_field"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); @@ -920,6 +936,9 @@ private XContentBuilder getMinMapping( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); b.endArray(); } if (!isEmptyMetrics) { @@ -951,6 +970,9 @@ private XContentBuilder getMinMapping( b.field("type", "integer"); b.endObject(); } + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1018,7 +1040,9 @@ private XContentBuilder getMinMappingWith2StarTrees() throws IOException { b.startObject("metric_field"); b.field("type", "integer"); b.endObject(); - + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1058,6 +1082,9 @@ private XContentBuilder getInvalidMapping( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); b.startArray("metrics"); @@ -1090,7 +1117,7 @@ private XContentBuilder getInvalidMapping( if (!invalidDimType) { b.field("type", "integer"); } else { - b.field("type", "keyword"); + b.field("type", "ip"); } b.endObject(); b.startObject("metric_field"); @@ -1100,6 +1127,9 @@ private XContentBuilder getInvalidMapping( b.field("type", "integer"); } b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1132,6 +1162,9 @@ private XContentBuilder getInvalidMappingWithDv( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); b.startArray("metrics"); @@ -1168,6 +1201,9 @@ private XContentBuilder getInvalidMappingWithDv( b.field("doc_values", "true"); } b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1224,6 +1260,9 @@ public void testEmptyName() { b.startObject("status"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }))); assertThat(e.getMessage(), containsString("name cannot be empty string")); diff --git a/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java index 0c88154ca2b38..3b32e9e4ac6b7 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java @@ -17,7 +17,6 @@ import org.opensearch.core.common.Strings; import org.opensearch.index.IndexService; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; -import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests; import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.mapper.CompositeMappedFieldType; @@ -26,6 +25,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.startree.StarTreeFilterTests; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.ReaderContext; @@ -55,7 +55,7 @@ public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { .indices() .prepareCreate("test") .setSettings(settings) - .setMapping(StarTreeDocValuesFormatTests.getExpandedMapping()); + .setMapping(StarTreeFilterTests.getExpandedMapping(1, false)); createIndex("test", builder); IndicesService indicesService = getInstanceFromNode(IndicesService.class); diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java index 0327bd9990784..12e83cbbadd5d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java @@ -90,7 +90,7 @@ protected Codec getCodec() { final Logger testLogger = LogManager.getLogger(MetricAggregatorTests.class); MapperService mapperService; try { - mapperService = StarTreeDocValuesFormatTests.createMapperService(StarTreeDocValuesFormatTests.getExpandedMapping()); + mapperService = StarTreeDocValuesFormatTests.createMapperService(StarTreeFilterTests.getExpandedMapping(1, false)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java index f8eb71a40319a..b03cb5ac7bb9d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java @@ -47,7 +47,7 @@ import java.util.List; import java.util.Map; -import static org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests.topMapping; +import static org.opensearch.index.codec.composite912.datacube.startree.AbstractStarTreeDVFormatTests.topMapping; public class StarTreeFilterTests extends AggregatorTestCase { From e9f77e38998bf42a17e34392bfa6683b1ec837d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:12:52 +0800 Subject: [PATCH 323/357] Bump org.apache.xmlbeans:xmlbeans from 5.2.1 to 5.2.2 in /plugins/ingest-attachment (#16612) * Bump org.apache.xmlbeans:xmlbeans in /plugins/ingest-attachment Bumps org.apache.xmlbeans:xmlbeans from 5.2.1 to 5.2.2. --- updated-dependencies: - dependency-name: org.apache.xmlbeans:xmlbeans dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/ingest-attachment/build.gradle | 2 +- plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1 | 1 - plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b1d5908c1a7..e8dd188709cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) +- Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612)) - Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611)) - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 81ac52b97cefa..4f30ea9ea7e22 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -89,7 +89,7 @@ dependencies { api "org.apache.poi:poi:${versions.poi}" api "org.apache.poi:poi-ooxml-lite:${versions.poi}" api "commons-codec:commons-codec:${versions.commonscodec}" - api 'org.apache.xmlbeans:xmlbeans:5.2.1' + api 'org.apache.xmlbeans:xmlbeans:5.2.2' api 'org.apache.commons:commons-collections4:4.4' // MS Office api "org.apache.poi:poi-scratchpad:${versions.poi}" diff --git a/plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1 b/plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1 deleted file mode 100644 index eaab556163e5c..0000000000000 --- a/plugins/ingest-attachment/licenses/xmlbeans-5.2.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e16ddf17fe181c202b097e0dcc0ee2fed91cb7da \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1 new file mode 100644 index 0000000000000..613c1028dbd6d --- /dev/null +++ b/plugins/ingest-attachment/licenses/xmlbeans-5.2.2.jar.sha1 @@ -0,0 +1 @@ +586ffe10ae9864e19e85c24bd060790a70586f72 \ No newline at end of file From 53d41d3fac28c9f72d5883467d5a6211ad09feac Mon Sep 17 00:00:00 2001 From: inpink <108166692+inpink@users.noreply.github.com> Date: Tue, 12 Nov 2024 21:47:36 +0900 Subject: [PATCH 324/357] feat: add vertical scaling and SoftReference for snapshot repository data cache (#16489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Applies `SoftReference` to cached repository data for efficient memory management under heap pressure. - Enables cache size configuration in `opensearch.yml`, adjustable within a range of 500KB to 1% of heap memory. - Sets the default cache size to `Math.max(ByteSizeUnit.KB.toBytes(500), CACHE_MAX_THRESHOLD / 2)` so it’s generally proportional to heap size. In cases where 1% of the heap is less than 1000KB, indicating a low-memory environment, the default reverts to 500KB as before. - Since `BytesReference` internally uses `byte[]`, the compressed array size is capped at `Integer.MAX_VALUE - 8` to ensure compatibility with JDK limitations on array sizes. Therefore, the maximum cache size cannot exceed this limit. Signed-off-by: inpink --- CHANGELOG.md | 1 + .../common/settings/ClusterSettings.java | 1 + .../blobstore/BlobStoreRepository.java | 111 ++++++++++++++++-- .../settings/MemorySizeSettingsTests.java | 63 +++++++++- .../blobstore/BlobStoreRepositoryTests.java | 50 ++++++++ 5 files changed, 209 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8dd188709cd2..515cf0ce93157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) +- Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index cac4b3914df5a..c836984655ad1 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -786,6 +786,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Snapshot related Settings BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING, BlobStoreRepository.SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, + BlobStoreRepository.SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 243d0021fac2e..c1305fa563b16 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -142,6 +142,7 @@ import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RecoveryState; +import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.IndexMetaDataGenerations; @@ -167,6 +168,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.ref.SoftReference; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Arrays; @@ -196,6 +198,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio; import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1; import static org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.canonicalName; import static org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat.SNAPSHOT_ONLY_FORMAT_PARAMS; @@ -253,6 +256,23 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp */ public static final String VIRTUAL_DATA_BLOB_PREFIX = "v__"; + public static final String SNAPSHOT_REPOSITORY_DATA_CACHET_THRESHOLD_SETTING_NAME = "snapshot.repository_data.cache.threshold"; + + public static final double SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD_DEFAULT_PERCENTAGE = 0.01; + + public static final long CACHE_MIN_THRESHOLD = ByteSizeUnit.KB.toBytes(500); + + public static final long CACHE_MAX_THRESHOLD = calculateMaxSnapshotRepositoryDataCacheThreshold(); + + public static final long CACHE_DEFAULT_THRESHOLD = calculateDefaultSnapshotRepositoryDataCacheThreshold(); + + /** + * Set to Integer.MAX_VALUE - 8 to prevent OutOfMemoryError due to array header requirements, following the limit used in certain JDK versions. + * This ensures compatibility across various JDK versions. For a practical usage example, + * see this link: https://github.com/openjdk/jdk11u/blob/cee8535a9d3de8558b4b5028d68e397e508bef71/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ByteArrayChannel.java#L226 + */ + private static final int MAX_SAFE_ARRAY_SIZE = Integer.MAX_VALUE - 8; + /** * When set to {@code true}, {@link #bestEffortConsistency} will be set to {@code true} and concurrent modifications of the repository * contents will not result in the repository being marked as corrupted. @@ -275,6 +295,58 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp Setting.Property.Deprecated ); + /** + * Sets the cache size for snapshot repository data: the valid range is within 500Kb ... 1% of the node heap memory. + */ + public static final Setting SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD = new Setting<>( + SNAPSHOT_REPOSITORY_DATA_CACHET_THRESHOLD_SETTING_NAME, + CACHE_DEFAULT_THRESHOLD + "b", + (s) -> { + ByteSizeValue userDefinedLimit = parseBytesSizeValueOrHeapRatio(s, SNAPSHOT_REPOSITORY_DATA_CACHET_THRESHOLD_SETTING_NAME); + long userDefinedLimitBytes = userDefinedLimit.getBytes(); + + if (userDefinedLimitBytes > CACHE_MAX_THRESHOLD) { + throw new IllegalArgumentException( + "[" + + SNAPSHOT_REPOSITORY_DATA_CACHET_THRESHOLD_SETTING_NAME + + "] cannot be larger than [" + + CACHE_MAX_THRESHOLD + + "] bytes." + ); + } + + if (userDefinedLimitBytes < CACHE_MIN_THRESHOLD) { + throw new IllegalArgumentException( + "[" + + SNAPSHOT_REPOSITORY_DATA_CACHET_THRESHOLD_SETTING_NAME + + "] cannot be smaller than [" + + CACHE_MIN_THRESHOLD + + "] bytes." + ); + } + + return userDefinedLimit; + }, + Setting.Property.NodeScope + ); + + public static long calculateDefaultSnapshotRepositoryDataCacheThreshold() { + return Math.max(ByteSizeUnit.KB.toBytes(500), CACHE_MAX_THRESHOLD / 2); + } + + public static long calculateMaxSnapshotRepositoryDataCacheThreshold() { + long jvmHeapSize = JvmInfo.jvmInfo().getMem().getHeapMax().getBytes(); + long defaultThresholdOfHeap = (long) (jvmHeapSize * SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD_DEFAULT_PERCENTAGE); + long defaultAbsoluteThreshold = ByteSizeUnit.KB.toBytes(500); + long maxThreshold = calculateMaxWithinIntLimit(defaultThresholdOfHeap, defaultAbsoluteThreshold); + + return maxThreshold; + } + + protected static long calculateMaxWithinIntLimit(long defaultThresholdOfHeap, long defaultAbsoluteThreshold) { + return Math.min(Math.max(defaultThresholdOfHeap, defaultAbsoluteThreshold), MAX_SAFE_ARRAY_SIZE); + } + /** * Size hint for the IO buffer size to use when reading from and writing to the repository. */ @@ -461,6 +533,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private volatile boolean enableAsyncDeletion; + protected final long repositoryDataCacheThreshold; + /** * Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for * {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart @@ -515,6 +589,7 @@ protected BlobStoreRepository( this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings()); this.enableAsyncDeletion = SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings().addSettingsUpdateConsumer(SNAPSHOT_ASYNC_DELETION_ENABLE_SETTING, this::setEnableAsyncDeletion); + this.repositoryDataCacheThreshold = SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD.get(clusterService.getSettings()).getBytes(); } @Override @@ -1132,7 +1207,8 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map> softRef = latestKnownRepositoryData.get(); + cached = (softRef != null) ? softRef.get() : null; } if (genToLoad > generation) { // It's always a possibility to not see the latest index-N in the listing here on an eventually consistent blob store, just @@ -2926,7 +3002,9 @@ public void endVerification(String seed) { private final AtomicLong latestKnownRepoGen = new AtomicLong(RepositoryData.UNKNOWN_REPO_GEN); // Best effort cache of the latest known repository data and its generation, cached serialized as compressed json - private final AtomicReference> latestKnownRepositoryData = new AtomicReference<>(); + private final AtomicReference>> latestKnownRepositoryData = new AtomicReference<>( + new SoftReference<>(null) + ); @Override public void getRepositoryData(ActionListener listener) { @@ -2934,7 +3012,9 @@ public void getRepositoryData(ActionListener listener) { listener.onFailure(corruptedStateException(null)); return; } - final Tuple cached = latestKnownRepositoryData.get(); + final SoftReference> softRef = latestKnownRepositoryData.get(); + final Tuple cached = (softRef != null) ? softRef.get() : null; + // Fast path loading repository data directly from cache if we're in fully consistent mode and the cache matches up with // the latest known repository generation if (bestEffortConsistency == false && cached != null && cached.v1() == latestKnownRepoGen.get()) { @@ -2983,7 +3063,8 @@ private void doGetRepositoryData(ActionListener listener) { genToLoad = latestKnownRepoGen.get(); } try { - final Tuple cached = latestKnownRepositoryData.get(); + final SoftReference> softRef = latestKnownRepositoryData.get(); + final Tuple cached = (softRef != null) ? softRef.get() : null; final RepositoryData loaded; // Caching is not used with #bestEffortConsistency see docs on #cacheRepositoryData for details if (bestEffortConsistency == false && cached != null && cached.v1() == genToLoad) { @@ -3050,19 +3131,22 @@ private void cacheRepositoryData(BytesReference updated, long generation) { try { serialized = CompressorRegistry.defaultCompressor().compress(updated); final int len = serialized.length(); - if (len > ByteSizeUnit.KB.toBytes(500)) { + long cacheWarningThreshold = Math.min(repositoryDataCacheThreshold * 10, MAX_SAFE_ARRAY_SIZE); + if (len > repositoryDataCacheThreshold) { logger.debug( - "Not caching repository data of size [{}] for repository [{}] because it is larger than 500KB in" + "Not caching repository data of size [{}] for repository [{}] because it is larger than [{}] bytes in" + " serialized size", len, - metadata.name() + metadata.name(), + repositoryDataCacheThreshold ); - if (len > ByteSizeUnit.MB.toBytes(5)) { + if (len > cacheWarningThreshold) { logger.warn( - "Your repository metadata blob for repository [{}] is larger than 5MB. Consider moving to a fresh" + "Your repository metadata blob for repository [{}] is larger than [{}] bytes. Consider moving to a fresh" + " repository for new snapshots or deleting unneeded snapshots from your repository to ensure stable" + " repository behavior going forward.", - metadata.name() + metadata.name(), + cacheWarningThreshold ); } // Set empty repository data to not waste heap for an outdated cached value @@ -3074,11 +3158,12 @@ private void cacheRepositoryData(BytesReference updated, long generation) { logger.warn("Failed to serialize repository data", e); return; } - latestKnownRepositoryData.updateAndGet(known -> { + latestKnownRepositoryData.updateAndGet(knownRef -> { + Tuple known = (knownRef != null) ? knownRef.get() : null; if (known != null && known.v1() > generation) { - return known; + return knownRef; } - return new Tuple<>(generation, serialized); + return new SoftReference<>(new Tuple<>(generation, serialized)); }); } } diff --git a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java index 95db7c2cfacaa..78782112be844 100644 --- a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java @@ -34,6 +34,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.indices.IndexingMemoryController; import org.opensearch.indices.IndicesQueryCache; @@ -41,6 +42,7 @@ import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.equalTo; @@ -127,22 +129,75 @@ public void testIndicesFieldDataCacheSetting() { ); } + public void testSnapshotRepositoryDataCacheSizeSetting() { + assertMemorySizeSettingInRange( + BlobStoreRepository.SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD, + "snapshot.repository_data.cache.threshold", + new ByteSizeValue(BlobStoreRepository.calculateDefaultSnapshotRepositoryDataCacheThreshold()), + ByteSizeUnit.KB.toBytes(500), + 1.0 + ); + } + private void assertMemorySizeSetting(Setting setting, String settingKey, ByteSizeValue defaultValue) { assertMemorySizeSetting(setting, settingKey, defaultValue, Settings.EMPTY); } private void assertMemorySizeSetting(Setting setting, String settingKey, ByteSizeValue defaultValue, Settings settings) { + assertMemorySizeSetting(setting, settingKey, defaultValue, 25.0, 1024, settings); + } + + private void assertMemorySizeSetting( + Setting setting, + String settingKey, + ByteSizeValue defaultValue, + double availablePercentage, + long availableBytes, + Settings settings + ) { assertThat(setting, notNullValue()); assertThat(setting.getKey(), equalTo(settingKey)); assertThat(setting.getProperties(), hasItem(Property.NodeScope)); assertThat(setting.getDefault(settings), equalTo(defaultValue)); - Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build(); + Settings settingWithPercentage = Settings.builder().put(settingKey, percentageAsString(availablePercentage)).build(); assertThat( setting.get(settingWithPercentage), - equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.25))) + equalTo( + new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * percentageAsFraction(availablePercentage))) + ) ); - Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build(); - assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024))); + Settings settingWithBytesValue = Settings.builder().put(settingKey, availableBytes + "b").build(); + assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(availableBytes))); } + private void assertMemorySizeSettingInRange( + Setting setting, + String settingKey, + ByteSizeValue defaultValue, + long minBytes, + double maxPercentage + ) { + assertMemorySizeSetting(setting, settingKey, defaultValue, maxPercentage, minBytes, Settings.EMPTY); + + assertThrows(IllegalArgumentException.class, () -> { + Settings settingWithTooSmallValue = Settings.builder().put(settingKey, minBytes - 1).build(); + setting.get(settingWithTooSmallValue); + }); + + assertThrows(IllegalArgumentException.class, () -> { + double unavailablePercentage = maxPercentage + 0.1; + Settings settingWithPercentageExceedingLimit = Settings.builder() + .put(settingKey, percentageAsString(unavailablePercentage)) + .build(); + setting.get(settingWithPercentageExceedingLimit); + }); + } + + private double percentageAsFraction(double availablePercentage) { + return availablePercentage / 100.0; + } + + private String percentageAsString(double availablePercentage) { + return availablePercentage + "%"; + } } diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index aa10b7dc18381..620b18ad9d7cf 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -92,6 +92,7 @@ import java.util.stream.Collectors; import static org.opensearch.repositories.RepositoryDataTests.generateRandomRepoData; +import static org.opensearch.repositories.blobstore.BlobStoreRepository.calculateMaxWithinIntLimit; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; @@ -653,4 +654,53 @@ public void testGetRestrictedSystemRepositorySettings() { assertTrue(settings.contains(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY)); repository.close(); } + + public void testSnapshotRepositoryDataCacheDefaultSetting() { + // given + BlobStoreRepository repository = setupRepo(); + long maxThreshold = BlobStoreRepository.calculateMaxSnapshotRepositoryDataCacheThreshold(); + + // when + long expectedThreshold = Math.max(ByteSizeUnit.KB.toBytes(500), maxThreshold / 2); + + // then + assertEquals(repository.repositoryDataCacheThreshold, expectedThreshold); + } + + public void testHeapThresholdUsed() { + // given + long defaultThresholdOfHeap = ByteSizeUnit.GB.toBytes(1); + long defaultAbsoluteThreshold = ByteSizeUnit.KB.toBytes(500); + + // when + long expectedThreshold = calculateMaxWithinIntLimit(defaultThresholdOfHeap, defaultAbsoluteThreshold); + + // then + assertEquals(defaultThresholdOfHeap, expectedThreshold); + } + + public void testAbsoluteThresholdUsed() { + // given + long defaultThresholdOfHeap = ByteSizeUnit.KB.toBytes(499); + long defaultAbsoluteThreshold = ByteSizeUnit.KB.toBytes(500); + + // when + long result = calculateMaxWithinIntLimit(defaultThresholdOfHeap, defaultAbsoluteThreshold); + + // then + assertEquals(defaultAbsoluteThreshold, result); + } + + public void testThresholdCappedAtIntMax() { + // given + int maxSafeArraySize = Integer.MAX_VALUE - 8; + long defaultThresholdOfHeap = (long) maxSafeArraySize + 1; + long defaultAbsoluteThreshold = ByteSizeUnit.KB.toBytes(500); + + // when + long expectedThreshold = calculateMaxWithinIntLimit(defaultThresholdOfHeap, defaultAbsoluteThreshold); + + // then + assertEquals(maxSafeArraySize, expectedThreshold); + } } From 5068fad53922f992ef651f78c148df6d9170c361 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 12 Nov 2024 10:52:41 -0800 Subject: [PATCH 325/357] Add a new configuration setting `synonym_analyzer` for `synonym_graph` and `synonym`. (#16488) * Add custom synonym_analyzer * synonym_analyzer configuration setting --------- Signed-off-by: Prudhvi Godithi --- CHANGELOG.md | 1 + .../common/CommonAnalysisModulePlugin.java | 29 +++++- .../SynonymGraphTokenFilterFactory.java | 11 ++- .../common/SynonymTokenFilterFactory.java | 28 +++++- .../common/CommonAnalysisFactoryTests.java | 22 +++++ .../common/SynonymsAnalysisTests.java | 95 +++++++++++++++++-- .../indices/analysis/AnalysisModule.java | 7 +- .../opensearch/plugins/AnalysisPlugin.java | 9 ++ .../indices/analysis/AnalysisModuleTests.java | 53 +++++++++++ .../analysis/AnalysisFactoryTestCase.java | 11 +++ 10 files changed, 250 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 515cf0ce93157..e46628249c91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) +- Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). ### Dependencies - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java index f14e499081ce9..7f9437972a358 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/CommonAnalysisModulePlugin.java @@ -146,6 +146,7 @@ import org.opensearch.index.analysis.PreConfiguredTokenizer; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.indices.analysis.AnalysisModule.AnalysisProvider; import org.opensearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import org.opensearch.plugins.AnalysisPlugin; @@ -247,7 +248,7 @@ public Map>> getAn } @Override - public Map> getTokenFilters() { + public Map> getTokenFilters(AnalysisModule analysisModule) { Map> filters = new TreeMap<>(); filters.put("apostrophe", ApostropheFilterFactory::new); filters.put("arabic_normalization", ArabicNormalizationFilterFactory::new); @@ -332,14 +333,36 @@ public Map> getTokenFilters() { filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new); filters.put("stemmer_override", requiresAnalysisSettings(StemmerOverrideTokenFilterFactory::new)); filters.put("stemmer", StemmerTokenFilterFactory::new); - filters.put("synonym", requiresAnalysisSettings(SynonymTokenFilterFactory::new)); - filters.put("synonym_graph", requiresAnalysisSettings(SynonymGraphTokenFilterFactory::new)); filters.put("trim", TrimTokenFilterFactory::new); filters.put("truncate", requiresAnalysisSettings(TruncateTokenFilterFactory::new)); filters.put("unique", UniqueTokenFilterFactory::new); filters.put("uppercase", UpperCaseTokenFilterFactory::new); filters.put("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new); filters.put("word_delimiter", WordDelimiterTokenFilterFactory::new); + filters.put( + "synonym", + requiresAnalysisSettings( + (indexSettings, environment, name, settings) -> new SynonymTokenFilterFactory( + indexSettings, + environment, + name, + settings, + analysisModule.getAnalysisRegistry() + ) + ) + ); + filters.put( + "synonym_graph", + requiresAnalysisSettings( + (indexSettings, environment, name, settings) -> new SynonymGraphTokenFilterFactory( + indexSettings, + environment, + name, + settings, + analysisModule.getAnalysisRegistry() + ) + ) + ); return filters; } diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymGraphTokenFilterFactory.java index fed959108c411..c2e20e99473de 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymGraphTokenFilterFactory.java @@ -40,6 +40,7 @@ import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisMode; +import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.CharFilterFactory; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; @@ -49,8 +50,14 @@ public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory { - SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(indexSettings, env, name, settings); + SynonymGraphTokenFilterFactory( + IndexSettings indexSettings, + Environment env, + String name, + Settings settings, + AnalysisRegistry analysisRegistry + ) { + super(indexSettings, env, name, settings, analysisRegistry); } @Override diff --git a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymTokenFilterFactory.java index 01a65e87d7466..1cd78170e66c8 100644 --- a/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/opensearch/analysis/common/SynonymTokenFilterFactory.java @@ -44,11 +44,13 @@ import org.opensearch.index.analysis.AbstractTokenFilterFactory; import org.opensearch.index.analysis.Analysis; import org.opensearch.index.analysis.AnalysisMode; +import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.CharFilterFactory; import org.opensearch.index.analysis.CustomAnalyzer; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.List; @@ -64,8 +66,16 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { protected final Settings settings; protected final Environment environment; protected final AnalysisMode analysisMode; - - SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + private final String synonymAnalyzerName; + private final AnalysisRegistry analysisRegistry; + + SynonymTokenFilterFactory( + IndexSettings indexSettings, + Environment env, + String name, + Settings settings, + AnalysisRegistry analysisRegistry + ) { super(indexSettings, name, settings); this.settings = settings; @@ -83,6 +93,8 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { boolean updateable = settings.getAsBoolean("updateable", false); this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; this.environment = env; + this.synonymAnalyzerName = settings.get("synonym_analyzer", null); + this.analysisRegistry = analysisRegistry; } @Override @@ -137,6 +149,17 @@ Analyzer buildSynonymAnalyzer( List tokenFilters, Function allFilters ) { + if (synonymAnalyzerName != null) { + Analyzer customSynonymAnalyzer; + try { + customSynonymAnalyzer = analysisRegistry.getAnalyzer(synonymAnalyzerName); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (customSynonymAnalyzer != null) { + return customSynonymAnalyzer; + } + } return new CustomAnalyzer( tokenizer, charFilters.toArray(new CharFilterFactory[0]), @@ -177,5 +200,4 @@ Reader getRulesFromSettings(Environment env) { } return rulesReader; } - } diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CommonAnalysisFactoryTests.java index 7e3140f8bcba3..1f4faf53dced5 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/CommonAnalysisFactoryTests.java @@ -39,12 +39,16 @@ import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory; import org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory; import org.apache.lucene.analysis.te.TeluguStemFilterFactory; +import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.indices.analysis.AnalysisFactoryTestCase; +import org.opensearch.indices.analysis.AnalysisModule; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.mockito.Mock; + import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; @@ -53,6 +57,9 @@ public CommonAnalysisFactoryTests() { super(new CommonAnalysisModulePlugin()); } + @Mock + private AnalysisModule analysisModule; + @Override protected Map> getTokenizers() { Map> tokenizers = new TreeMap<>(super.getTokenizers()); @@ -302,4 +309,19 @@ private void markedTestCase(String name, Map> map) { unmarked ); } + + /** + * Tests the getTokenFilters(AnalysisModule) method to verify: + * 1. All token filters are properly loaded + * 2. Basic filters remain available + * 3. Synonym filters remain available when AnalysisModule is provided + */ + public void testGetTokenFiltersWithAnalysisModule() { + CommonAnalysisModulePlugin plugin = (CommonAnalysisModulePlugin) getAnalysisPlugin(); + Map> filters = plugin.getTokenFilters(analysisModule); + assertNotNull("Token filters should not be null", filters); + assertTrue("Should contain basic filters", filters.containsKey("lowercase")); + assertTrue("Should contain synonym filter", filters.containsKey("synonym")); + assertTrue("Should contain synonym_graph filter", filters.containsKey("synonym_graph")); + } } diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java index 8c8b8ac7f61c0..33d92e01a85b1 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java @@ -41,11 +41,14 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; +import org.opensearch.env.TestEnvironment; import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.PreConfiguredTokenFilter; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -63,6 +66,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; +import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; public class SynonymsAnalysisTests extends OpenSearchTestCase { private IndexAnalyzers indexAnalyzers; @@ -255,14 +259,16 @@ public void testTokenFiltersBypassSynonymAnalysis() throws IOException { .put("hyphenation_patterns_path", "foo") .build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); - + Environment environment = TestEnvironment.newEnvironment(settings); + AnalysisModule analysisModule = new AnalysisModule(environment, Collections.singletonList(new CommonAnalysisModulePlugin())); + AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry(); String[] bypassingFactories = new String[] { "dictionary_decompounder" }; CommonAnalysisModulePlugin plugin = new CommonAnalysisModulePlugin(); for (String factory : bypassingFactories) { - TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings); - TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings); - SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings); + TokenFilterFactory tff = plugin.getTokenFilters(analysisModule).get(factory).get(idxSettings, environment, factory, settings); + TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, environment, "keyword", settings); + SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, environment, "synonym", settings, analysisRegistry); Analyzer analyzer = stff.buildSynonymAnalyzer(tok, Collections.emptyList(), Collections.singletonList(tff), null); try (TokenStream ts = analyzer.tokenStream("field", "text")) { @@ -319,7 +325,11 @@ public void testDisallowedTokenFilters() throws IOException { .putList("common_words", "a", "b") .put("output_unigrams", "true") .build(); + + Environment environment = TestEnvironment.newEnvironment(settings); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); + AnalysisModule analysisModule = new AnalysisModule(environment, Collections.singletonList(new CommonAnalysisModulePlugin())); + AnalysisRegistry analysisRegistry = analysisModule.getAnalysisRegistry(); CommonAnalysisModulePlugin plugin = new CommonAnalysisModulePlugin(); String[] disallowedFactories = new String[] { @@ -333,9 +343,9 @@ public void testDisallowedTokenFilters() throws IOException { "fingerprint" }; for (String factory : disallowedFactories) { - TokenFilterFactory tff = plugin.getTokenFilters().get(factory).get(idxSettings, null, factory, settings); - TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, null, "keyword", settings); - SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, null, "synonym", settings); + TokenFilterFactory tff = plugin.getTokenFilters(analysisModule).get(factory).get(idxSettings, environment, factory, settings); + TokenizerFactory tok = new KeywordTokenizerFactory(idxSettings, environment, "keyword", settings); + SynonymTokenFilterFactory stff = new SynonymTokenFilterFactory(idxSettings, environment, "synonym", settings, analysisRegistry); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -362,4 +372,75 @@ private void match(String analyzerName, String source, String target) throws IOE MatcherAssert.assertThat(target, equalTo(sb.toString().trim())); } + /** + * Tests the integration of word delimiter and synonym graph filters with synonym_analyzer based on issue #16263. + * This test verifies the correct handling of: + * 1. Hyphenated words with word delimiter (e.g., "note-book" → ["notebook", "note", "book"]) + * 2. Multi-word synonyms (e.g., "mobile phone" → ["smartphone"]) + * 3. Single word synonyms (e.g., "laptop" → ["notebook"]) + * + * @see Issue #16263 + */ + public void testSynonymAnalyzerWithWordDelimiter() throws IOException { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put("path.home", createTempDir().toString()) + .put("index.analysis.filter.custom_word_delimiter.type", "word_delimiter_graph") + .put("index.analysis.filter.custom_word_delimiter.generate_word_parts", true) + .put("index.analysis.filter.custom_word_delimiter.catenate_all", true) + .put("index.analysis.filter.custom_word_delimiter.split_on_numerics", false) + .put("index.analysis.filter.custom_word_delimiter.split_on_case_change", false) + .put("index.analysis.filter.custom_pattern_replace_filter.type", "pattern_replace") + .put("index.analysis.filter.custom_pattern_replace_filter.pattern", "(-)") + .put("index.analysis.filter.custom_pattern_replace_filter.replacement", " ") + .put("index.analysis.filter.custom_pattern_replace_filter.all", true) + .put("index.analysis.filter.custom_synonym_graph_filter.type", "synonym_graph") + .putList( + "index.analysis.filter.custom_synonym_graph_filter.synonyms", + "laptop => notebook", + "smartphone, mobile phone, cell phone => smartphone", + "tv, television => television" + ) + .put("index.analysis.filter.custom_synonym_graph_filter.synonym_analyzer", "standard") + .put("index.analysis.analyzer.text_en_index.type", "custom") + .put("index.analysis.analyzer.text_en_index.tokenizer", "whitespace") + .putList( + "index.analysis.analyzer.text_en_index.filter", + "lowercase", + "custom_word_delimiter", + "custom_synonym_graph_filter", + "custom_pattern_replace_filter", + "flatten_graph" + ) + .build(); + Environment environment = TestEnvironment.newEnvironment(settings); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings); + AnalysisModule module = new AnalysisModule(environment, Collections.singletonList(new CommonAnalysisModulePlugin())); + IndexAnalyzers analyzers = module.getAnalysisRegistry().build(indexSettings); + try (TokenStream ts = analyzers.get("text_en_index").tokenStream("", "note-book")) { + assertTokenStreamContents( + ts, + new String[] { "notebook", "note", "book" }, + new int[] { 0, 0, 5 }, + new int[] { 9, 4, 9 }, + new String[] { "word", "word", "word" }, + new int[] { 1, 0, 1 }, + new int[] { 2, 1, 1 } + ); + } + try (TokenStream ts = analyzers.get("text_en_index").tokenStream("", "mobile phone")) { + assertTokenStreamContents( + ts, + new String[] { "smartphone" }, + new int[] { 0 }, + new int[] { 12 }, + new String[] { "SYNONYM" }, + new int[] { 1 }, + new int[] { 1 } + ); + } + try (TokenStream ts = analyzers.get("text_en_index").tokenStream("", "laptop")) { + assertTokenStreamContents(ts, new String[] { "notebook" }, new int[] { 0 }, new int[] { 6 }); + } + } } diff --git a/server/src/main/java/org/opensearch/indices/analysis/AnalysisModule.java b/server/src/main/java/org/opensearch/indices/analysis/AnalysisModule.java index 0926d497087d1..dbb3035a18f74 100644 --- a/server/src/main/java/org/opensearch/indices/analysis/AnalysisModule.java +++ b/server/src/main/java/org/opensearch/indices/analysis/AnalysisModule.java @@ -165,7 +165,12 @@ public boolean requiresAnalysisSettings() { ) ); - tokenFilters.extractAndRegister(plugins, AnalysisPlugin::getTokenFilters); + for (AnalysisPlugin plugin : plugins) { + Map> filters = plugin.getTokenFilters(this); + for (Map.Entry> entry : filters.entrySet()) { + tokenFilters.register(entry.getKey(), entry.getValue()); + } + } return tokenFilters; } diff --git a/server/src/main/java/org/opensearch/plugins/AnalysisPlugin.java b/server/src/main/java/org/opensearch/plugins/AnalysisPlugin.java index 53dcc916b244f..58e43633777c9 100644 --- a/server/src/main/java/org/opensearch/plugins/AnalysisPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/AnalysisPlugin.java @@ -47,6 +47,7 @@ import org.opensearch.index.analysis.PreConfiguredTokenizer; import org.opensearch.index.analysis.TokenFilterFactory; import org.opensearch.index.analysis.TokenizerFactory; +import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.indices.analysis.AnalysisModule.AnalysisProvider; import java.io.IOException; @@ -84,6 +85,14 @@ default Map> getCharFilters() { return emptyMap(); } + /** + * Override to add additional {@link TokenFilter}s that need access to the AnalysisModule. + * The default implementation for plugins that don't need AnalysisModule calls the existing getTokenFilters() method. + */ + default Map> getTokenFilters(AnalysisModule analysisModule) { + return getTokenFilters(); + } + /** * Override to add additional {@link TokenFilter}s. See {@link #requiresAnalysisSettings(AnalysisProvider)} * how to on get the configuration from the index. diff --git a/server/src/test/java/org/opensearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/opensearch/indices/analysis/AnalysisModuleTests.java index c9e26d6d6159a..74bc987c44b15 100644 --- a/server/src/test/java/org/opensearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/opensearch/indices/analysis/AnalysisModuleTests.java @@ -56,6 +56,8 @@ import org.opensearch.index.analysis.CustomAnalyzer; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.MyFilterTokenFilterFactory; +import org.opensearch.index.analysis.NameOrDefinition; +import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.analysis.PreConfiguredCharFilter; import org.opensearch.index.analysis.PreConfiguredTokenFilter; import org.opensearch.index.analysis.PreConfiguredTokenizer; @@ -80,6 +82,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -521,4 +524,54 @@ public boolean incrementToken() throws IOException { } } + /** + * Tests registration and functionality of token filters that require access to the AnalysisModule. + * This test verifies the token filter registration using the extended getTokenFilters(AnalysisModule) method + */ + public void testTokenFilterRegistrationWithModuleReference() throws IOException { + class TestPlugin implements AnalysisPlugin { + @Override + public Map> getTokenFilters(AnalysisModule module) { + return Map.of( + "test_filter", + (indexSettings, env, name, settings) -> AppendTokenFilter.factoryForSuffix("_" + module.hashCode()) + ); + } + } + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard") + .put("index.analysis.analyzer.my_analyzer.filter", "test_filter") + .build(); + Environment environment = TestEnvironment.newEnvironment(settings); + AnalysisModule module = new AnalysisModule(environment, singletonList(new TestPlugin())); + AnalysisRegistry registry = module.getAnalysisRegistry(); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder().put(settings).build()); + Map tokenFilterFactories = registry.buildTokenFilterFactories(indexSettings); + assertTrue("Token filter 'test_filter' should be registered", tokenFilterFactories.containsKey("test_filter")); + IndexAnalyzers analyzers = registry.build(indexSettings); + String testText = "test"; + TokenStream tokenStream = analyzers.get("my_analyzer").tokenStream("", testText); + CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); + tokenStream.reset(); + assertTrue("Should have found a token", tokenStream.incrementToken()); + assertEquals("Token should have expected suffix", "test_" + module.hashCode(), charTermAttribute.toString()); + assertFalse("Should not have additional tokens", tokenStream.incrementToken()); + tokenStream.close(); + NamedAnalyzer customAnalyzer = registry.buildCustomAnalyzer( + indexSettings, + false, + new NameOrDefinition("standard"), + Collections.emptyList(), + Collections.singletonList(new NameOrDefinition("test_filter")) + ); + tokenStream = customAnalyzer.tokenStream("", testText); + charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); + tokenStream.reset(); + assertTrue("Custom analyzer should produce a token", tokenStream.incrementToken()); + assertEquals("Custom analyzer token should have expected suffix", "test_" + module.hashCode(), charTermAttribute.toString()); + assertFalse("Custom analyzer should not produce additional tokens", tokenStream.incrementToken()); + tokenStream.close(); + } } diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java index 23cf4d47a49d9..ca23f67215f3d 100644 --- a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java @@ -248,6 +248,17 @@ public AnalysisFactoryTestCase(AnalysisPlugin plugin) { this.plugin = Objects.requireNonNull(plugin, "plugin is required. use an empty plugin for core"); } + /** + * Returns the AnalysisPlugin instance that was passed to this test case. + * This protected method allows subclasses to access the plugin for testing + * specific analysis components. + * + * @return The AnalysisPlugin instance used by this test case + */ + protected AnalysisPlugin getAnalysisPlugin() { + return plugin; + } + protected Map> getCharFilters() { return KNOWN_CHARFILTERS; } From 548a6505d6eaa88170462ef5eaf45a89a367126e Mon Sep 17 00:00:00 2001 From: rajiv-kv <157019998+rajiv-kv@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:20:23 +0530 Subject: [PATCH 326/357] Support prefix list for remote repository attributes (#16271) * Support prefix list for remote repository attributes Signed-off-by: Rajiv Kumar Vaidyanathan --- CHANGELOG.md | 1 + .../RemotePublicationConfigurationIT.java | 274 +--------------- .../RemoteRepositoryConfigurationIT.java | 308 ++++++++++++++++++ .../coordination/JoinTaskExecutor.java | 19 +- .../metadata/MetadataCreateIndexService.java | 8 +- .../cluster/node/DiscoveryNode.java | 32 +- .../InternalRemoteRoutingTableService.java | 5 +- .../remote/RemoteClusterStateService.java | 6 +- .../index/remote/RemoteIndexPathUploader.java | 20 +- .../RemoteMigrationIndexMetadataUpdater.java | 8 +- .../remotestore/RemoteStoreNodeAttribute.java | 248 +++++++++++--- .../RemoteStorePinnedTimestampService.java | 7 +- .../cluster/node/DiscoveryNodeTests.java | 2 +- .../remote/RemoteIndexPathUploaderTests.java | 14 +- .../node/RemoteStoreNodeAttributeTests.java | 54 +++ .../test/OpenSearchIntegTestCase.java | 2 +- .../test/RemoteStoreAttributeConstants.java | 19 ++ 17 files changed, 635 insertions(+), 392 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java create mode 100644 test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e46628249c91e..c19019ece6c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) +- Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271)) - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). ### Dependencies diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java index 57bf9eccbf5b4..1b5d924fa0b62 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java @@ -8,34 +8,9 @@ package org.opensearch.gateway.remote; -import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; -import org.opensearch.common.settings.Settings; -import org.opensearch.plugins.Plugin; -import org.opensearch.remotemigration.MigrationBaseTestCase; -import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.fs.FsRepository; -import org.opensearch.repositories.fs.ReloadableFsRepository; -import org.opensearch.test.InternalSettingsPlugin; import org.opensearch.test.OpenSearchIntegTestCase; -import org.opensearch.test.transport.MockTransportService; -import org.junit.Assert; import org.junit.Before; -import java.util.Collection; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - /** * Tests the compatibility between types of nodes based on the configured repositories * Non Remote node [No Repositories configured] @@ -44,260 +19,15 @@ * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] */ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class RemotePublicationConfigurationIT extends MigrationBaseTestCase { - private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; - - @Override - protected Collection> nodePlugins() { - /* Adding the following mock plugins: - - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync - - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated - */ - return Stream.concat( - super.nodePlugins().stream(), - Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) - ).collect(Collectors.toList()); - } - +public class RemotePublicationConfigurationIT extends RemoteRepositoryConfigurationIT { @Before public void setUp() throws Exception { if (segmentRepoPath == null || translogRepoPath == null) { segmentRepoPath = randomRepoPath().toAbsolutePath(); translogRepoPath = randomRepoPath().toAbsolutePath(); } + super.remoteRepoPrefix = "remote_publication"; super.setUp(); } - public Settings.Builder remotePublishConfiguredNodeSetting() { - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_NAME - ); - String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_NAME - ); - String routingTableRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - ROUTING_TABLE_REPO_NAME - ); - String routingTableRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - ROUTING_TABLE_REPO_NAME - ); - - Settings.Builder builder = Settings.builder() - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) - .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) - .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME) - .put(routingTableRepoTypeAttributeKey, FsRepository.TYPE) - .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); - return builder; - } - - public Settings.Builder remoteWithRoutingTableNodeSetting() { - // Remote Cluster with Routing table - - return Settings.builder() - .put( - remoteStoreClusterSettings( - REPOSITORY_NAME, - segmentRepoPath, - ReloadableFsRepository.TYPE, - REPOSITORY_2_NAME, - translogRepoPath, - ReloadableFsRepository.TYPE, - REPOSITORY_NAME, - segmentRepoPath, - ReloadableFsRepository.TYPE - ) - ) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); - } - - public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - - ensureStableCluster(3); - ensureGreen(); - - internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); - } - - public void testServiceInitialized_WhenNodeAttributesPresent() { - internalCluster().startClusterManagerOnlyNode( - buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) - ); - internalCluster().startDataOnlyNodes( - 2, - buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) - ); - - ensureStableCluster(3); - ensureGreen(); - - internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); - } - - public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ensureStableCluster(6); - ensureGreen(); - } - - public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { - // Remote Cluster without Routing table - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - setAddRemote(false); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNodes(2); - setAddRemote(false); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - // Remote Repo with Routing table - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - ensureStableCluster(4); - ensureGreen(); - } - - public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - internalCluster().startDataOnlyNodes(2, settings); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - internalCluster().startClusterManagerOnlyNode(); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { - Settings settings = remoteWithRoutingTableNodeSetting().build(); - internalCluster().startClusterManagerOnlyNode(settings); - internalCluster().startDataOnlyNodes(2, settings); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); - - ensureStableCluster(4); - ensureGreen(); - } - - public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - internalCluster().startClusterManagerOnlyNode(); - - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - - setAddRemote(true); - internalCluster().startClusterManagerOnlyNode(); - ensureStableCluster(4); - ensureGreen(); - } - - public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { - Settings.Builder build = remotePublishConfiguredNodeSetting(); - internalCluster().startClusterManagerOnlyNode(build.build()); - internalCluster().startDataOnlyNodes(2, build.build()); - - ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); - updateSettingsRequest.persistentSettings( - Settings.builder() - .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") - .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") - ); - - Settings settings = Settings.builder() - .put( - buildRemoteStoreNodeAttributes( - REPOSITORY_NAME, - segmentRepoPath, - REPOSITORY_2_NAME, - translogRepoPath, - ROUTING_TABLE_REPO_NAME, - segmentRepoPath, - false - ) - ) - .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .build(); - assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); - internalCluster().startClusterManagerOnlyNode(settings); - - ensureStableCluster(4); - ensureGreen(); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java new file mode 100644 index 0000000000000..48afa85dc5691 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRepositoryConfigurationIT.java @@ -0,0 +1,308 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.remotemigration.MigrationBaseTestCase; +import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.repositories.fs.ReloadableFsRepository; +import org.opensearch.test.InternalSettingsPlugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.junit.Assert; +import org.junit.Before; + +import java.util.Collection; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * Tests the compatibility between types of nodes based on the configured repositories + * Non Remote node [No Repositories configured] + * Remote Publish Configured Node [Cluster State + Routing Table] + * Remote Node [Cluster State + Segment + Translog] + * Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table] + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteRepositoryConfigurationIT extends MigrationBaseTestCase { + private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica"; + + protected String remoteRepoPrefix = "remote_store"; + + @Override + protected Collection> nodePlugins() { + /* Adding the following mock plugins: + - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync + - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated + */ + return Stream.concat( + super.nodePlugins().stream(), + Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class) + ).collect(Collectors.toList()); + } + + @Before + public void setUp() throws Exception { + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } + super.setUp(); + } + + public Settings.Builder remotePublishConfiguredNodeSetting() { + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + remoteRepoPrefix, + REPOSITORY_NAME + ); + String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey(); + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + remoteRepoPrefix, + REPOSITORY_NAME + ); + String routingTableRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + remoteRepoPrefix, + ROUTING_TABLE_REPO_NAME + ); + String routingTableRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + remoteRepoPrefix, + ROUTING_TABLE_REPO_NAME + ); + + Settings.Builder builder = Settings.builder() + .put("node.attr." + remoteRepoPrefix + ".state.repository", REPOSITORY_NAME) + .put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + remoteRepoPrefix + ".routing_table.repository", ROUTING_TABLE_REPO_NAME) + .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE) + .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); + return builder; + } + + public Settings.Builder remoteWithRoutingTableNodeSetting() { + // Remote Cluster with Routing table + return Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + REPOSITORY_2_NAME, + translogRepoPath, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true); + } + + public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull); + } + + public void testServiceInitialized_WhenNodeAttributesPresent() { + internalCluster().startClusterManagerOnlyNode( + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + internalCluster().startDataOnlyNodes( + 2, + buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE) + ); + + ensureStableCluster(3); + ensureGreen(); + + internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull); + } + + public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ensureStableCluster(6); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception { + // Remote Cluster without Routing table + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception { + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + setAddRemote(false); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + // Remote Repo with Routing table + Settings settings = remoteWithRoutingTableNodeSetting().build(); + + internalCluster().startClusterManagerOnlyNode(settings); + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception { + Settings settings = remoteWithRoutingTableNodeSetting().build(); + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build()); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + internalCluster().startClusterManagerOnlyNode(); + + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + setAddRemote(true); + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(4); + ensureGreen(); + } + + public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception { + Settings.Builder build = remotePublishConfiguredNodeSetting(); + internalCluster().startClusterManagerOnlyNode(build.build()); + internalCluster().startDataOnlyNodes(2, build.build()); + + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings( + Settings.builder() + .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store") + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed") + ); + + Settings settings = Settings.builder() + .put( + buildRemoteStoreNodeAttributes( + REPOSITORY_NAME, + segmentRepoPath, + ReloadableFsRepository.TYPE, + REPOSITORY_2_NAME, + translogRepoPath, + FsRepository.TYPE, + ROUTING_TABLE_REPO_NAME, + segmentRepoPath, + ReloadableFsRepository.TYPE, + false + ) + ) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + internalCluster().startClusterManagerOnlyNode(settings); + + ensureStableCluster(4); + ensureGreen(); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index d597b51c32ccd..ec30496a3f7ad 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -55,6 +55,7 @@ import org.opensearch.persistent.PersistentTasksCustomMetadata; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -69,7 +70,8 @@ import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getClusterStateRepoName; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRoutingTableRepoName; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED; import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; @@ -539,7 +541,12 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi .findFirst(); if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationEnabled()) { - ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES); + List repos = Arrays.asList( + getClusterStateRepoName(remotePublicationNode.get().getAttributes()), + getRoutingTableRepoName(remotePublicationNode.get().getAttributes()) + ); + + ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), repos); } } @@ -568,16 +575,12 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod List reposToSkip = new ArrayList<>(1); // find a remote node which has routing table configured Optional remoteRoutingTableNode = existingNodes.stream() - .filter( - node -> node.isRemoteStoreNode() - && node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null - ) + .filter(node -> node.isRemoteStoreNode() && RemoteStoreNodeAttribute.getRoutingTableRepoName(node.getAttributes()) != null) .findFirst(); // If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node. // This ensures a new node with remote routing table repo is able to join the cluster. if (remoteRoutingTableNode.isEmpty()) { - String joiningNodeRepoName = joiningNode.getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY); + String joiningNodeRepoName = getRoutingTableRepoName(joiningNode.getAttributes()); if (joiningNodeRepoName != null) { reposToSkip.add(joiningNodeRepoName); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 727a08b615050..232201d18ba13 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1178,12 +1178,8 @@ public static void updateRemoteStoreSettings( .findFirst(); if (remoteNode.isPresent()) { - translogRepo = remoteNode.get() - .getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); - segmentRepo = remoteNode.get() - .getAttributes() - .get(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes()); + segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes()); if (segmentRepo != null && translogRepo != null) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo) diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 8c9a37a767ede..d84fb794c5e4f 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -45,6 +45,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import java.io.IOException; import java.util.Collections; @@ -62,10 +63,9 @@ import java.util.stream.Stream; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isClusterStateRepoConfigured; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRoutingTableRepoConfigured; /** * A discovery node represents a node that is part of the cluster. @@ -510,8 +510,7 @@ public boolean isSearchNode() { * @return true if the node contains remote store node attributes, false otherwise */ public boolean isRemoteStoreNode() { - return this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) - && this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return isClusterStateRepoConfigured(this.getAttributes()) && RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes()); } /** @@ -519,11 +518,7 @@ public boolean isRemoteStoreNode() { * @return true if the node contains remote cluster state node attribute and remote routing table node attribute */ public boolean isRemoteStatePublicationEnabled() { - return this.getAttributes() - .keySet() - .stream() - .anyMatch(key -> (key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY))) - && this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return isClusterStateRepoConfigured(this.getAttributes()) && isRoutingTableRepoConfigured(this.getAttributes()); } /** @@ -587,13 +582,16 @@ public String toString() { sb.append('}'); } if (!attributes.isEmpty()) { - sb.append( - attributes.entrySet() - .stream() - .filter(entry -> !entry.getKey().startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)) // filter remote_store attributes - // from logging to reduce noise. - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + sb.append(attributes.entrySet().stream().filter(entry -> { + for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) { + if (entry.getKey().startsWith(prefix)) { + return false; + } + } + return true; + }) // filter remote_store attributes + // from logging to reduce noise. + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } return sb.toString(); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index ea8f980c14972..eafbe05faf76f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -33,7 +33,6 @@ import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable; import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff; import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -235,9 +234,7 @@ protected void doClose() throws IOException { @Override protected void doStart() { assert isRemoteRoutingTableConfigured(settings) == true : "Remote routing table is not enabled"; - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = RemoteStoreNodeAttribute.getRoutingTableRepoName(settings); assert remoteStoreRepo != null : "Remote routing table repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index dc41189afc3cb..e4f4bae9bef7c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -61,7 +61,6 @@ import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff; import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -1065,9 +1064,8 @@ public void close() throws IOException { public void start() { assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled"; - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = RemoteStoreNodeAttribute.getClusterStateRepoName(settings); + assert remoteStoreRepo != null : "Remote Cluster State repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 2a76a5b966884..18b6d6184d1b0 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -26,7 +26,6 @@ import org.opensearch.gateway.remote.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.RemoteStoreSettings; -import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; @@ -70,11 +69,6 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener { private static final String TIMEOUT_EXCEPTION_MSG = "Timed out waiting while uploading remote index path file for indexes=%s"; private static final String UPLOAD_EXCEPTION_MSG = "Exception occurred while uploading remote index paths for indexes=%s"; - static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() - + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; - static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() - + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; - private static final Logger logger = LogManager.getLogger(RemoteIndexPathUploader.class); private final Settings settings; @@ -226,9 +220,8 @@ private void writePathToRemoteStore( } } - private Repository validateAndGetRepository(String repoSetting) { - final String repo = settings.get(repoSetting); - assert repo != null : "Remote " + repoSetting + " repository is not configured"; + private Repository validateAndGetRepository(String repo) { + assert repo != null : "Remote repository is not configured"; final Repository repository = repositoriesService.get().repository(repo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; return repository; @@ -240,15 +233,16 @@ public void start() { // If remote store data attributes are not present than we skip this. return; } - translogRepository = (BlobStoreRepository) validateAndGetRepository(TRANSLOG_REPO_NAME_KEY); - segmentRepository = (BlobStoreRepository) validateAndGetRepository(SEGMENT_REPO_NAME_KEY); + + translogRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings)); + segmentRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings)); } private boolean isTranslogSegmentRepoSame() { // TODO - The current comparison checks the repository name. But it is also possible that the repository are same // by attributes, but different by name. We need to handle this. - String translogRepoName = settings.get(TRANSLOG_REPO_NAME_KEY); - String segmentRepoName = settings.get(SEGMENT_REPO_NAME_KEY); + String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings); + String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings); return Objects.equals(translogRepoName, segmentRepoName); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java index cc51fcd2f18f6..1f9ffca4460b7 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java @@ -18,6 +18,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import java.util.List; import java.util.Map; @@ -30,8 +31,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStoreCustomMetadataDuringMigration; import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; /** * Utils for checking and mutating cluster state during remote migration @@ -74,8 +73,9 @@ public void maybeAddRemoteIndexSettings(IndexMetadata.Builder indexMetadataBuild index ); Map remoteRepoNames = getRemoteStoreRepoName(discoveryNodes); - String segmentRepoName = remoteRepoNames.get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); - String tlogRepoName = remoteRepoNames.get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + String segmentRepoName = RemoteStoreNodeAttribute.getSegmentRepoName(remoteRepoNames); + String tlogRepoName = RemoteStoreNodeAttribute.getTranslogRepoName(remoteRepoNames); + assert Objects.nonNull(segmentRepoName) && Objects.nonNull(tlogRepoName) : "Remote repo names cannot be null"; Settings.Builder indexSettingsBuilder = Settings.builder().put(currentIndexSettings); updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, tlogRepoName); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index b1b6259e4ca18..89d06753063b7 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,12 +12,14 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -35,10 +37,28 @@ */ public class RemoteStoreNodeAttribute { - public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; + public static final List REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = List.of("remote_store", "remote_publication"); + + // TO-DO the string constants are used only for tests and can be moved to test package + public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; - public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; + + public static final List REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".state.repository") + .collect(Collectors.toList()); + + public static final List REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".routing_table.repository") + .collect(Collectors.toList()); + public static final List REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".segment.repository") + .collect(Collectors.toList()); + public static final List REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() + .map(prefix -> prefix + ".translog.repository") + .collect(Collectors.toList()); + public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; @@ -46,18 +66,19 @@ public class RemoteStoreNodeAttribute { + "." + CryptoMetadata.SETTINGS_KEY; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; - public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; - private final RepositoriesMetadata repositoriesMetadata; + public static final String REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s.type"; + public static final String REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "%s.repository.%s." + CryptoMetadata.CRYPTO_METADATA_KEY; + public static final String REPOSITORY_CRYPTO_SETTINGS_PREFIX = REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT + + "." + + CryptoMetadata.SETTINGS_KEY; + public static final String REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "%s.repository.%s.settings."; - public static List SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = List.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + private final RepositoriesMetadata repositoriesMetadata; - public static List REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES = List.of( - REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, - REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + public static List> SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = Arrays.asList( + REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS, + REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS ); /** @@ -76,8 +97,17 @@ private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) return attributeValue; } - private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repositoryName) { - String metadataKey = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repositoryName); + private Tuple validateAttributeNonNull(DiscoveryNode node, List attributeKeys) { + Tuple attributeValue = getValue(node.getAttributes(), attributeKeys); + if (attributeValue == null || attributeValue.v1() == null || attributeValue.v1().isEmpty()) { + throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKeys.get(0) + "]"); + } + + return attributeValue; + } + + private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repositoryName, String prefix) { + String metadataKey = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, prefix, repositoryName); boolean isRepoEncrypted = node.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(metadataKey)); if (isRepoEncrypted == false) { return null; @@ -86,11 +116,7 @@ private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repository String keyProviderName = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_NAME_KEY); String keyProviderType = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_TYPE_KEY); - String settingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, - repositoryName - ); + String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_SETTINGS_PREFIX, prefix, repositoryName); Map settingsMap = node.getAttributes() .keySet() @@ -104,10 +130,11 @@ private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repository return new CryptoMetadata(keyProviderName, keyProviderType, settings.build()); } - private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName) { + private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName, String prefix) { String settingsAttributeKeyPrefix = String.format( Locale.getDefault(), - REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + prefix, repositoryName ); Map settingsMap = node.getAttributes() @@ -125,17 +152,17 @@ private Map validateSettingsAttributesNonNull(DiscoveryNode node return settingsMap; } - private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name, String prefix) { String type = validateAttributeNonNull( node, - String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name) + String.format(Locale.getDefault(), REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, prefix, name) ); - Map settingsMap = validateSettingsAttributesNonNull(node, name); + Map settingsMap = validateSettingsAttributesNonNull(node, name, prefix); Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); - CryptoMetadata cryptoMetadata = buildCryptoMetadata(node, name); + CryptoMetadata cryptoMetadata = buildCryptoMetadata(node, name, prefix); // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); @@ -144,53 +171,104 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na } private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { - Set repositoryNames = getValidatedRepositoryNames(node); + Map repositoryNamesWithPrefix = getValidatedRepositoryNames(node); List repositoryMetadataList = new ArrayList<>(); - for (String repositoryName : repositoryNames) { - repositoryMetadataList.add(buildRepositoryMetadata(node, repositoryName)); + for (Map.Entry repository : repositoryNamesWithPrefix.entrySet()) { + repositoryMetadataList.add(buildRepositoryMetadata(node, repository.getKey(), repository.getValue())); } return new RepositoriesMetadata(repositoryMetadataList); } - private Set getValidatedRepositoryNames(DiscoveryNode node) { - Set repositoryNames = new HashSet<>(); - if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) - || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); - } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + private static Tuple getValue(Map attributes, List keys) { + for (String key : keys) { + if (attributes.containsKey(key)) { + return new Tuple<>(attributes.get(key), key); + } } - if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + return null; + } + + private Map getValidatedRepositoryNames(DiscoveryNode node) { + Set> repositoryNames = new HashSet<>(); + if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) + || containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + } else if (containsKey(node.getAttributes(), REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + } + if (containsKey(node.getAttributes(), REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); } - return repositoryNames; + Map repoNamesWithPrefix = new HashMap<>(); + repositoryNames.forEach(t -> { + String[] attrKeyParts = t.v2().split("\\."); + repoNamesWithPrefix.put(t.v1(), attrKeyParts[0]); + }); + + return repoNamesWithPrefix; } public static boolean isRemoteStoreAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX).isEmpty() == false; + for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteDataAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false - || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; + return isSegmentRepoConfigured(settings) || isTranslogRepoConfigured(settings); + } + + public static boolean isSegmentRepoConfigured(Settings settings) { + for (String prefix : REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; + } + + public static boolean isTranslogRepoConfigured(Settings settings) { + for (String prefix : REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteClusterStateConfigured(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) - .isEmpty() == false; + for (String prefix : REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static String getRemoteStoreSegmentRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + for (String prefix : REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); + } + } + return null; } public static String getRemoteStoreTranslogRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + for (String prefix : REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); + } + } + return null; } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { @@ -198,8 +276,12 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { } private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) - .isEmpty() == false; + for (String prefix : REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS) { + if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { + return true; + } + } + return false; } public static boolean isRemoteRoutingTableConfigured(Settings settings) { @@ -219,21 +301,83 @@ public RepositoriesMetadata getRepositoriesMetadata() { public static Map getDataRepoNames(DiscoveryNode node) { assert remoteDataAttributesPresent(node.getAttributes()); Map dataRepoNames = new HashMap<>(); - for (String supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { - dataRepoNames.put(supportedRepoAttribute, node.getAttributes().get(supportedRepoAttribute)); + for (List supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { + Tuple value = getValue(node.getAttributes(), supportedRepoAttribute); + if (value != null && value.v1() != null) { + dataRepoNames.put(value.v2(), value.v1()); + } } return dataRepoNames; } private static boolean remoteDataAttributesPresent(Map nodeAttrs) { - for (String supportedRepoAttributes : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { - if (nodeAttrs.get(supportedRepoAttributes) == null || nodeAttrs.get(supportedRepoAttributes).isEmpty()) { + for (List supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) { + Tuple value = getValue(nodeAttrs, supportedRepoAttribute); + if (value == null || value.v1() == null) { return false; } } return true; } + public static String getClusterStateRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getRoutingTableRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getSegmentRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getTranslogRepoName(Map repos) { + return getValueFromAnyKey(repos, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static String getValueFromAnyKey(Map repos, List keys) { + for (String key : keys) { + if (repos.get(key) != null) { + return repos.get(key); + } + } + return null; + } + + public static String getClusterStateRepoName(Settings settings) { + return getValueFromAnyKey(settings, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static String getRoutingTableRepoName(Settings settings) { + return getValueFromAnyKey(settings, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static String getValueFromAnyKey(Settings settings, List keys) { + for (String key : keys) { + if (settings.get(Node.NODE_ATTRIBUTES.getKey() + key) != null) { + return settings.get(Node.NODE_ATTRIBUTES.getKey() + key); + } + } + return null; + } + + public static boolean isClusterStateRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static boolean isRoutingTableRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + public static boolean isSegmentRepoConfigured(Map attributes) { + return containsKey(attributes, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS); + } + + private static boolean containsKey(Map attributes, List keys) { + return keys.stream().filter(k -> attributes.containsKey(k)).findFirst().isPresent(); + } + @Override public int hashCode() { // The hashCode is generated by computing the hash of all the repositoryMetadata present in diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java index 98fcad0e6c496..f5b372ddd9b80 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStorePinnedTimestampService.java @@ -21,7 +21,6 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.action.ActionListener; import org.opensearch.indices.RemoteStoreSettings; -import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -42,6 +41,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo; + /** * Service for managing pinned timestamps in a remote store. * This service handles pinning and unpinning of timestamps, as well as periodic updates of the pinned timestamps set. @@ -86,9 +87,7 @@ public void start() { } private static BlobContainer validateAndCreateBlobContainer(Settings settings, RepositoriesService repositoriesService) { - final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - ); + final String remoteStoreRepo = getRemoteStoreSegmentRepo(settings); assert remoteStoreRepo != null : "Remote Segment Store repository is not configured"; final Repository repository = repositoriesService.repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index 525a53f3e6158..6550ed39e8042 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -99,7 +99,7 @@ public void testRemoteStoreRedactionInToString() { roles, Version.CURRENT ); - assertFalse(node.toString().contains(RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)); + assertFalse(node.toString().contains(RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.get(0))); } public void testDiscoveryNodeIsCreatedWithHostFromInetAddress() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java index d6519d9db8ee6..2e6523a4a64a0 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java @@ -80,11 +80,16 @@ public class RemoteIndexPathUploaderTests extends OpenSearchTestCase { private final AtomicLong successCount = new AtomicLong(); private final AtomicLong failureCount = new AtomicLong(); + static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() + + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; + static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey() + + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + @Before public void setup() { settings = Settings.builder() - .put(RemoteIndexPathUploader.TRANSLOG_REPO_NAME_KEY, TRANSLOG_REPO_NAME) - .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, TRANSLOG_REPO_NAME) + .put(TRANSLOG_REPO_NAME_KEY, TRANSLOG_REPO_NAME) + .put(SEGMENT_REPO_NAME_KEY, TRANSLOG_REPO_NAME) .put(CLUSTER_STATE_REPO_KEY, TRANSLOG_REPO_NAME) .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) .build(); @@ -247,10 +252,7 @@ public void testInterceptWithSameRepo() throws IOException { } public void testInterceptWithDifferentRepo() throws IOException { - Settings settings = Settings.builder() - .put(this.settings) - .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, SEGMENT_REPO_NAME) - .build(); + Settings settings = Settings.builder().put(this.settings).put(SEGMENT_REPO_NAME_KEY, SEGMENT_REPO_NAME).build(); when(repositoriesService.repository(SEGMENT_REPO_NAME)).thenReturn(repository); RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader( threadPool, diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index de7f8977686a7..537a5a5739b75 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -32,12 +32,66 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_CRYPTO_SETTINGS_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.test.RemoteStoreAttributeConstants.REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { static private final String KEY_ARN = "arn:aws:kms:us-east-1:123456789:key/6e9aa906-2cc3-4924-8ded-f385c78d9dcf"; static private final String REGION = "us-east-1"; + public void testCryptoMetadataForPublication() throws UnknownHostException { + String repoName = "remote-store-A"; + String prefix = "remote_publication"; + String repoTypeSettingKey = String.format(Locale.ROOT, REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, prefix, repoName); + String repoSettingsKey = String.format(Locale.ROOT, REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, prefix, repoName); + String repoCryptoMetadataKey = String.format(Locale.ROOT, REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, prefix, repoName); + String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REPOSITORY_CRYPTO_SETTINGS_PREFIX, prefix, repoName); + Map attr = Map.of( + REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, + repoName, + repoTypeSettingKey, + "s3", + repoSettingsKey, + "abc", + repoSettingsKey + "base_path", + "xyz", + repoCryptoMetadataKey + ".key_provider_name", + "store-test", + repoCryptoMetadataKey + ".key_provider_type", + "aws-kms", + repoCryptoMetadataSettingsKey + ".region", + REGION, + repoCryptoMetadataSettingsKey + ".key_arn", + KEY_ARN + ); + DiscoveryNode node = new DiscoveryNode( + "C", + new TransportAddress(InetAddress.getByName("localhost"), 9876), + attr, + emptySet(), + Version.CURRENT + ); + + RemoteStoreNodeAttribute remoteStoreNodeAttribute = new RemoteStoreNodeAttribute(node); + assertEquals(remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().size(), 1); + RepositoryMetadata repositoryMetadata = remoteStoreNodeAttribute.getRepositoriesMetadata().repositories().get(0); + Settings.Builder settings = Settings.builder(); + settings.put("region", REGION); + settings.put("key_arn", KEY_ARN); + CryptoMetadata cryptoMetadata = new CryptoMetadata("store-test", "aws-kms", settings.build()); + assertEquals(cryptoMetadata, repositoryMetadata.cryptoMetadata()); + } + public void testCryptoMetadata() throws UnknownHostException { String repoName = "remote-store-A"; String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index e27ff311c06f6..1ee856d3092f0 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2838,7 +2838,7 @@ private static Settings buildRemoteStoreNodeAttributes( ); } - private static Settings buildRemoteStoreNodeAttributes( + protected static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, String segmentRepoType, diff --git a/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java b/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java new file mode 100644 index 0000000000000..0e7ebb9f871f6 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/RemoteStoreAttributeConstants.java @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.test; + +public class RemoteStoreAttributeConstants { + + public static final String REMOTE_PUBLICATION_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.state.repository"; + public static final String REMOTE_PUBLICATION_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.segment.repository"; + public static final String REMOTE_PUBLICATION_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_publication.translog.repository"; + public static final String REMOTE_PUBLICATION_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = + "remote_publication.routing_table.repository"; + +} From 3f18562974612ceb36b66c773b6b8eb6d4dfc0b0 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Thu, 14 Nov 2024 13:12:55 +0800 Subject: [PATCH 327/357] Bump google-auth-library-oauth2-http from 1.7.0 to 1.29.0 in /plugins/repository-gcs (#16520) * Bump from 1.7.0 to 1.29.0 in /plugins/repository-gcs Signed-off-by: Gao Binlong * Add change log Signed-off-by: Gao Binlong * move version to top-level file Signed-off-by: Gao Binlong * Fix build failure Signed-off-by: Gao Binlong * Fix test failure Signed-off-by: Gao Binlong * Revert some change Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong Signed-off-by: gaobinlong --- CHANGELOG.md | 1 + gradle/libs.versions.toml | 1 + plugins/repository-gcs/build.gradle | 12 +++++++----- .../google-auth-library-credentials-1.29.0.jar.sha1 | 1 + .../google-auth-library-credentials-1.7.0.jar.sha1 | 1 - .../google-auth-library-oauth2-http-1.29.0.jar.sha1 | 1 + .../google-auth-library-oauth2-http-1.7.0.jar.sha1 | 1 - .../gcs/GoogleCloudStorageServiceTests.java | 4 ++-- 8 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index c19019ece6c5c..0a5aca03abd0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). ### Dependencies +- Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520)) - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8ff3f6e45397d..6b5c10e071d1e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ guava = "32.1.1-jre" protobuf = "3.25.5" jakarta_annotation = "1.3.5" google_http_client = "1.44.1" +google_auth = "1.29.0" tdigest = "3.3" hdrhistogram = "2.2.2" grpc = "1.68.0" diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index b90bcc7f822d1..a44f6643c086a 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -47,10 +47,6 @@ opensearchplugin { classname 'org.opensearch.repositories.gcs.GoogleCloudStoragePlugin' } -versions << [ - 'google_auth': '1.7.0' -] - dependencies { api 'com.google.api:api-common:1.8.1' api 'com.google.api:gax:2.35.0' @@ -148,7 +144,6 @@ thirdPartyAudit { 'com.google.appengine.api.urlfetch.HTTPResponse', 'com.google.appengine.api.urlfetch.URLFetchService', 'com.google.appengine.api.urlfetch.URLFetchServiceFactory', - 'com.google.auth.oauth2.GdchCredentials', 'com.google.protobuf.util.JsonFormat', 'com.google.protobuf.util.JsonFormat$Parser', 'com.google.protobuf.util.JsonFormat$Printer', @@ -304,6 +299,13 @@ testClusters { all testClustersConfiguration } +/** + * Used for testing getting credentials from GCE + */ +test { + environment 'NO_GCE_CHECK', 'true' +} + /* * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 new file mode 100644 index 0000000000000..e2f931a1e876f --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.29.0.jar.sha1 @@ -0,0 +1 @@ +19af4907301816d9328c1eb1fcc6dd05c8a0b544 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 deleted file mode 100644 index f2e9a4f7283bf..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b29af5a9ea94e9e7f86bded11e39f5afda5b17e8 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 new file mode 100644 index 0000000000000..98d0d1beda43d --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.29.0.jar.sha1 @@ -0,0 +1 @@ +2a42aead6cdc5d2cd22cdda1b9d7922e6135240f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 deleted file mode 100644 index 738645d6b8c7b..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -985d183303dbd4b7ceb348056e41e59677f6f74f \ No newline at end of file diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 58e412684ed5a..b620f212df413 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -242,7 +242,7 @@ public void testApplicationDefaultCredentialsWhenNoSettingProvided() throws Exce Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault); assertNotNull(storageOptions); assertNull(storageOptions.getCredentials()); - MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available")); + MatcherAssert.assertThat(exception.getMessage(), containsString("Your default credentials were not found")); } /** @@ -254,7 +254,7 @@ public void testDefaultCredentialsThrowsExceptionWithoutGCStorageService() { GoogleCredentials credentials = googleApplicationDefaultCredentials.get(); assertNull(credentials); Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault); - MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available")); + MatcherAssert.assertThat(exception.getMessage(), containsString("Your default credentials were not found")); } /** From 4cce608bffbef4978bfc2e13c91a9fe594f4cf2d Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Thu, 14 Nov 2024 10:45:59 +0530 Subject: [PATCH 328/357] Fix stale index deletion in snapshots for hashed prefix path type (#16617) Signed-off-by: Ashish Singh --- .../snapshots/DeleteSnapshotIT.java | 31 ++++++++++++ .../blobstore/BlobStoreRepository.java | 50 +++++++++++++------ .../snapshots/SnapshotShardPaths.java | 15 ++++-- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 26b30af4c2c50..85d81761ea4a0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -45,6 +45,37 @@ public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String REMOTE_REPO_NAME = "remote-store-repo-name"; + public void testStaleIndexDeletion() throws Exception { + String indexName1 = ".testindex1"; + String repoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath = randomRepoPath().toAbsolutePath(); + logger.info("Path [{}]", absolutePath); + + Client client = client(); + // Write a document + String docId = Integer.toString(randomInt()); + index(indexName1, "_doc", docId, "value", "expected"); + createRepository(repoName, "fs", absolutePath); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName1) + .setWaitForCompletion(true) + .setIndices(indexName1) + .get(); + assertTrue(createSnapshotResponse.getSnapshotInfo().successfulShards() > 0); + assertEquals(createSnapshotResponse.getSnapshotInfo().totalShards(), createSnapshotResponse.getSnapshotInfo().successfulShards()); + assertEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state()); + + assertAcked(startDeleteSnapshot(repoName, snapshotName1).get()); + assertBusy(() -> assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath.resolve(BlobStoreRepository.INDICES_DIR)))); + assertBusy(() -> assertEquals(0, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath.resolve(SnapshotShardPaths.DIR)))); + // At the end there are 2 files that exists - index-N and index.latest + assertBusy(() -> assertEquals(2, RemoteStoreBaseIntegTestCase.getFileCount(absolutePath))); + } + public void testDeleteSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); final Path remoteStoreRepoPath = randomRepoPath(); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index c1305fa563b16..a82c05dab0b44 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2383,11 +2383,23 @@ private List findMatchingShardPaths(String indexId, Map findHighestGenerationShardPaths(List matchingShardPaths) { - return matchingShardPaths.stream() - .map(s -> s.split("\\" + SnapshotShardPaths.DELIMITER)) - .sorted((a, b) -> Integer.parseInt(b[2]) - Integer.parseInt(a[2])) - .map(parts -> String.join(SnapshotShardPaths.DELIMITER, parts)) - .findFirst(); + if (matchingShardPaths.isEmpty()) { + return Optional.empty(); + } + + int maxGen = Integer.MIN_VALUE; + String maxGenShardPath = null; + + for (String shardPath : matchingShardPaths) { + String[] parts = shardPath.split("\\" + SnapshotShardPaths.DELIMITER); + int shardCount = Integer.parseInt(parts[parts.length - 3]); + if (shardCount > maxGen) { + maxGen = shardCount; + maxGenShardPath = shardPath; + } + } + assert maxGenShardPath != null : "Valid maxGenShardPath should be present"; + return Optional.of(maxGenShardPath); } /** @@ -2625,22 +2637,28 @@ public void finalizeSnapshot( * on account of new indexes by same index name being snapshotted that exists already in the repository's snapshots. */ private void cleanupRedundantSnapshotShardPaths(Set updatedShardPathsIndexIds) { - Set updatedIndexIds = updatedShardPathsIndexIds.stream() - .map(s -> getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0])) - .collect(Collectors.toSet()); - Set indexIdShardPaths = getSnapshotShardPaths().keySet(); - List staleShardPaths = indexIdShardPaths.stream().filter(s -> updatedShardPathsIndexIds.contains(s) == false).filter(s -> { - String indexId = getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0]); - return updatedIndexIds.contains(indexId); - }).collect(Collectors.toList()); try { + Set updatedIndexIds = updatedShardPathsIndexIds.stream() + .map(s -> getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0])) + .collect(Collectors.toSet()); + logger.debug(new ParameterizedMessage("updatedIndexIds={}", updatedIndexIds)); + Set indexIdShardPaths = getSnapshotShardPaths().keySet(); + logger.debug(new ParameterizedMessage("indexIdShardPaths={}", indexIdShardPaths)); + List staleShardPaths = indexIdShardPaths.stream() + .filter(s -> updatedShardPathsIndexIds.contains(s) == false) + .filter(s -> { + String indexId = getIndexId(s.split("\\" + SnapshotShardPaths.DELIMITER)[0]); + return updatedIndexIds.contains(indexId); + }) + .collect(Collectors.toList()); + logger.debug(new ParameterizedMessage("staleShardPaths={}", staleShardPaths)); deleteFromContainer(snapshotShardPathBlobContainer(), staleShardPaths); - } catch (IOException e) { + } catch (Exception e) { logger.warn( new ParameterizedMessage( - "Repository [{}] Exception during snapshot stale index deletion {}", + "Repository [{}] Exception during snapshot stale index deletion for updatedIndexIds {}", metadata.name(), - staleShardPaths + updatedShardPathsIndexIds ), e ); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java index 878c2baba4ce9..dd0b67ca9bfaa 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardPaths.java @@ -92,18 +92,25 @@ public static SnapshotShardPaths fromXContent(XContentParser ignored) { * Parses a shard path string and extracts relevant shard information. * * @param shardPath The shard path string to parse. Expected format is: - * [index_id]#[index_name]#[shard_count]#[path_type_code]#[path_hash_algorithm_code] + * snapshot_path_[index_id].[index_name].[shard_count].[path_type_code].[path_hash_algorithm_code] * @return A {@link ShardInfo} object containing the parsed index ID and shard count. * @throws IllegalArgumentException if the shard path format is invalid or cannot be parsed. */ public static ShardInfo parseShardPath(String shardPath) { String[] parts = shardPath.split("\\" + SnapshotShardPaths.DELIMITER); - if (parts.length != 5) { + int len = parts.length; + if (len < 5) { throw new IllegalArgumentException("Invalid shard path format: " + shardPath); } try { - IndexId indexId = new IndexId(parts[1], getIndexId(parts[0]), Integer.parseInt(parts[3])); - int shardCount = Integer.parseInt(parts[2]); + String indexName = shardPath.substring( + // First separator after index id + shardPath.indexOf(DELIMITER) + 1, + // Since we know there are exactly 3 fields at the end + shardPath.lastIndexOf(DELIMITER, shardPath.lastIndexOf(DELIMITER, shardPath.lastIndexOf(DELIMITER) - 1) - 1) + ); + IndexId indexId = new IndexId(indexName, getIndexId(parts[0]), Integer.parseInt(parts[len - 2])); + int shardCount = Integer.parseInt(parts[len - 3]); return new ShardInfo(indexId, shardCount); } catch (NumberFormatException e) { throw new IllegalArgumentException("Invalid shard path format: " + shardPath, e); From ac3e4ff0115602fe7b0f149e7d6976626324e019 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:40:04 +0800 Subject: [PATCH 329/357] Bump com.google.cloud:google-cloud-core-http from 2.23.0 to 2.47.0 in /plugins/repository-gcs (#16504) * Bump com.google.cloud:google-cloud-core-http in /plugins/repository-gcs Bumps com.google.cloud:google-cloud-core-http from 2.23.0 to 2.47.0. --- updated-dependencies: - dependency-name: com.google.cloud:google-cloud-core-http dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Signed-off-by: gaobinlong Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: gaobinlong --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 2 +- .../licenses/google-cloud-core-http-2.23.0.jar.sha1 | 1 - .../licenses/google-cloud-core-http-2.47.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5aca03abd0c..dbcbbaaa9435c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). ### Dependencies +- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520)) - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) - Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index a44f6643c086a..97ae88aac5485 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -63,7 +63,7 @@ dependencies { api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" api 'com.google.cloud:google-cloud-core:2.30.0' - api 'com.google.cloud:google-cloud-core-http:2.23.0' + api 'com.google.cloud:google-cloud-core-http:2.47.0' api 'com.google.cloud:google-cloud-storage:1.113.1' api 'com.google.code.gson:gson:2.11.0' diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 deleted file mode 100644 index 9db3cbcbec35b..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.23.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9913d0806fcfbfbc4a775f29865126ed8465464b \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 new file mode 100644 index 0000000000000..224893caeaafb --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.47.0.jar.sha1 @@ -0,0 +1 @@ +bfc8c587e8f2f1f1158cf36b0e515ef84f9e0a95 \ No newline at end of file From 3b9ca63351829a7e8435050a98f787d0e9afa37e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 14 Nov 2024 14:57:26 -0500 Subject: [PATCH 330/357] Support installing plugin SNAPSHOTs with SNASPHOT distribution (#16581) * Support installing plugin SNAPSHOTs with SNASPHOT distribution Signed-off-by: Andriy Redko * Use Build.CURRENT directly for snapshots Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../plugins/InstallPluginCommand.java | 30 +--- .../plugins/InstallPluginCommandTests.java | 165 ++---------------- 3 files changed, 21 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbcbbaaa9435c..5991d9593e259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) - Increase segrep pressure checkpoint default limit to 30 ([#16577](https://github.com/opensearch-project/OpenSearch/pull/16577/files)) - Add dynamic setting allowing size > 0 requests to be cached in the request cache ([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) +- Support installing plugin SNAPSHOTs with SNASPHOT distribution ([#16581](https://github.com/opensearch-project/OpenSearch/pull/16581)) - Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583)) - Support for keyword fields in star-tree index ([#16233](https://github.com/opensearch-project/OpenSearch/pull/16233)) - Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600)) diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java index 838d6e22a37bd..511d6974085aa 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java @@ -137,8 +137,6 @@ */ class InstallPluginCommand extends EnvironmentAwareCommand { - private static final String PROPERTY_STAGING_ID = "opensearch.plugins.staging"; - // exit codes for install /** A plugin with the same name is already installed. */ static final int PLUGIN_EXISTS = 1; @@ -307,14 +305,7 @@ void execute(Terminal terminal, List pluginIds, boolean isBatch, Environ private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean isBatch) throws Exception { if (OFFICIAL_PLUGINS.contains(pluginId)) { - final String url = getOpenSearchUrl( - terminal, - getStagingHash(), - Version.CURRENT, - isSnapshot(), - pluginId, - Platforms.PLATFORM_NAME - ); + final String url = getOpenSearchUrl(terminal, Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME); terminal.println("-> Downloading " + pluginId + " from opensearch"); return downloadAndValidate(terminal, url, tmpDir, true, isBatch); } @@ -341,11 +332,6 @@ private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean i return downloadZip(terminal, pluginId, tmpDir, isBatch); } - // pkg private so tests can override - String getStagingHash() { - return System.getProperty(PROPERTY_STAGING_ID); - } - boolean isSnapshot() { return Build.CURRENT.isSnapshot(); } @@ -353,26 +339,18 @@ boolean isSnapshot() { /** Returns the url for an official opensearch plugin. */ private String getOpenSearchUrl( final Terminal terminal, - final String stagingHash, final Version version, final boolean isSnapshot, final String pluginId, final String platform ) throws IOException, UserException { final String baseUrl; - if (isSnapshot && stagingHash == null) { - throw new UserException( - ExitCodes.CONFIG, - "attempted to install release build of official plugin on snapshot build of OpenSearch" - ); - } - if (stagingHash != null) { + if (isSnapshot == true) { baseUrl = String.format( Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/%s/%s-%s", + "https://artifacts.opensearch.org/snapshots/plugins/%s/%s", pluginId, - version, - stagingHash + Build.CURRENT.getQualifiedVersion() ); } else { baseUrl = String.format( diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java index c264788df20e8..e6c1070f36066 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java @@ -990,7 +990,6 @@ void assertInstallPluginFromUrl( final String pluginId, final String name, final String url, - final String stagingHash, final boolean isSnapshot, final String shaExtension, final Function shaCalculator, @@ -1065,11 +1064,6 @@ boolean urlExists(Terminal terminal, String urlString) throws IOException { return urlString.equals(url); } - @Override - String getStagingHash() { - return stagingHash; - } - @Override boolean isSnapshot() { return isSnapshot; @@ -1084,19 +1078,13 @@ void jarHellCheck(PluginInfo candidateInfo, Path candidate, Path pluginsDir, Pat assertPlugin(name, pluginDir, env.v2()); } - public void assertInstallPluginFromUrl( - final String pluginId, - final String name, - final String url, - final String stagingHash, - boolean isSnapshot - ) throws Exception { + public void assertInstallPluginFromUrl(final String pluginId, final String name, final String url, boolean isSnapshot) + throws Exception { final MessageDigest digest = MessageDigest.getInstance("SHA-512"); assertInstallPluginFromUrl( pluginId, name, url, - stagingHash, isSnapshot, ".sha512", checksumAndFilename(digest, url), @@ -1111,42 +1099,17 @@ public void testOfficialPlugin() throws Exception { + "/analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false); } public void testOfficialPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s.zip", - Version.CURRENT, - Build.CURRENT.getQualifiedVersion() - ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); - } - - public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { - String url = String.format( - Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s.zip", + "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-SNAPSHOT/analysis-icu-%s.zip", Version.CURRENT, Build.CURRENT.getQualifiedVersion() ); - // attemping to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception - final UserException e = expectThrows( - UserException.class, - () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, true) - ); - assertThat(e.exitCode, equalTo(ExitCodes.CONFIG)); - assertThat(e, hasToString(containsString("attempted to install release build of official plugin on snapshot build of OpenSearch"))); - } - - public void testOfficialPluginStaging() throws Exception { - String url = "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/" - + Version.CURRENT - + "-abc123/analysis-icu-" - + Build.CURRENT.getQualifiedVersion() - + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, true); } public void testOfficialPlatformPlugin() throws Exception { @@ -1157,62 +1120,30 @@ public void testOfficialPlatformPlugin() throws Exception { + "-" + Build.CURRENT.getQualifiedVersion() + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, null, false); - } - - public void testOfficialPlatformPluginSnapshot() throws Exception { - String url = String.format( - Locale.ROOT, - "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/%s-abc123/analysis-icu-%s-%s.zip", - Version.CURRENT, - Platforms.PLATFORM_NAME, - Build.CURRENT.getQualifiedVersion() - ); - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", true); - } - - public void testOfficialPlatformPluginStaging() throws Exception { - String url = "https://artifacts.opensearch.org/snapshots/plugins/analysis-icu/" - + Version.CURRENT - + "-abc123/analysis-icu-" - + Platforms.PLATFORM_NAME - + "-" - + Build.CURRENT.getQualifiedVersion() - + ".zip"; - assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, "abc123", false); + assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false); } public void testMavenPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false); } public void testMavenPlatformPlugin() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-" + Platforms.PLATFORM_NAME + "-1.0.0.zip"; - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false); } public void testMavenSha1Backcompat() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-1"); - assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, null, false, ".sha1", checksum(digest), null, (b, p) -> null); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".sha1", checksum(digest), null, (b, p) -> null); assertTrue(terminal.getOutput(), terminal.getOutput().contains("sha512 not found, falling back to sha1")); } public void testMavenChecksumWithoutFilename() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; MessageDigest digest = MessageDigest.getInstance("SHA-512"); - assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", - "myplugin", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ); + assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".sha512", checksum(digest), null, (b, p) -> null); } public void testOfficialChecksumWithoutFilename() throws Exception { @@ -1224,17 +1155,7 @@ public void testOfficialChecksumWithoutFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertThat(e.getMessage(), startsWith("Invalid checksum file")); @@ -1249,17 +1170,7 @@ public void testOfficialShaMissing() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-1"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha1", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha1", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha512", e.getMessage()); @@ -1269,17 +1180,7 @@ public void testMavenShaMissing() throws Exception { String url = "https://repo1.maven.org/maven2/mygroup/myplugin/1.0.0/myplugin-1.0.0.zip"; UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "mygroup:myplugin:1.0.0", - "myplugin", - url, - null, - false, - ".dne", - bytes -> null, - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("mygroup:myplugin:1.0.0", "myplugin", url, false, ".dne", bytes -> null, null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertEquals("Plugin checksum missing: " + url + ".sha1", e.getMessage()); @@ -1294,17 +1195,7 @@ public void testInvalidShaFileMissingFilename() throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-512"); UserException e = expectThrows( UserException.class, - () -> assertInstallPluginFromUrl( - "analysis-icu", - "analysis-icu", - url, - null, - false, - ".sha512", - checksum(digest), - null, - (b, p) -> null - ) + () -> assertInstallPluginFromUrl("analysis-icu", "analysis-icu", url, false, ".sha512", checksum(digest), null, (b, p) -> null) ); assertEquals(ExitCodes.IO_ERROR, e.exitCode); assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid checksum file")); @@ -1323,7 +1214,6 @@ public void testInvalidShaFileMismatchFilename() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", checksumAndString(digest, " repository-s3-" + Build.CURRENT.getQualifiedVersion() + ".zip"), @@ -1348,7 +1238,6 @@ public void testInvalidShaFileContainingExtraLine() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", checksumAndString(digest, " analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip\nfoobar"), @@ -1372,7 +1261,6 @@ public void testSha512Mismatch() throws Exception { "analysis-icu", "analysis-icu", url, - null, false, ".sha512", bytes -> "foobar analysis-icu-" + Build.CURRENT.getQualifiedVersion() + ".zip", @@ -1392,7 +1280,6 @@ public void testSha1Mismatch() throws Exception { "mygroup:myplugin:1.0.0", "myplugin", url, - null, false, ".sha1", bytes -> "foobar", @@ -1426,17 +1313,7 @@ public void testPublicKeyIdMismatchToExpectedPublicKeyId() throws Exception { final String expectedID = Long.toHexString(verifyingKey.getKeyID()).toUpperCase(Locale.ROOT); final IllegalStateException e = expectThrows( IllegalStateException.class, - () -> assertInstallPluginFromUrl( - icu, - icu, - url, - null, - false, - ".sha512", - checksumAndFilename(digest, url), - verifyingKey, - signature - ) + () -> assertInstallPluginFromUrl(icu, icu, url, false, ".sha512", checksumAndFilename(digest, url), verifyingKey, signature) ); assertThat(e, hasToString(containsString("key id [" + actualID + "] does not match expected key id [" + expectedID + "]"))); } @@ -1463,17 +1340,7 @@ public void testFailedSignatureVerification() throws Exception { }; final IllegalStateException e = expectThrows( IllegalStateException.class, - () -> assertInstallPluginFromUrl( - icu, - icu, - url, - null, - false, - ".sha512", - checksumAndFilename(digest, url), - newSecretKey(), - signature - ) + () -> assertInstallPluginFromUrl(icu, icu, url, false, ".sha512", checksumAndFilename(digest, url), newSecretKey(), signature) ); assertThat(e, hasToString(equalTo("java.lang.IllegalStateException: signature verification for [" + url + "] failed"))); } From 26e6ea1df0deda786e29231ed1fa2b567da18ea4 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 14 Nov 2024 17:45:51 -0500 Subject: [PATCH 331/357] Update DEVELOPER_GUIDE.md instructions for JDK-11 (#16533) * Update DEVELOPER_GUIDE.md instructions for JDK-11 Signed-off-by: Andriy Redko * Address review comments Signed-off-by: Andriy Redko * Update DEVELOPER_GUIDE.md Co-authored-by: Andrew Ross Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko Co-authored-by: Andrew Ross --- DEVELOPER_GUIDE.md | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 03cd189aa911e..b40f5d9b3f21a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2,9 +2,7 @@ - [Getting Started](#getting-started) - [Git Clone OpenSearch Repo](#git-clone-opensearch-repo) - [Install Prerequisites](#install-prerequisites) - - [JDK 11](#jdk-11) - - [JDK 14](#jdk-14) - - [JDK 17](#jdk-17) + - [JDK](#jdk) - [Custom Runtime JDK](#custom-runtime-jdk) - [Windows](#windows) - [Docker](#docker) @@ -76,35 +74,24 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS ### Install Prerequisites -#### JDK 11 +#### JDK -OpenSearch builds using Java 11 at a minimum, using the Adoptium distribution. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. This is configured in [buildSrc/build.gradle](buildSrc/build.gradle) and [distribution/tools/java-version-checker/build.gradle](distribution/tools/java-version-checker/build.gradle). +OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-23 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. -``` -allprojects { - targetCompatibility = JavaVersion.VERSION_11 - sourceCompatibility = JavaVersion.VERSION_11 -} -``` +Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). -``` -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 -``` -Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). +In addition, certain backward compatibility tests check out and compile the previous major version of OpenSearch, and therefore require installing [JDK 11](https://adoptium.net/temurin/releases/?version=11) and [JDK 17](https://adoptium.net/temurin/releases/?version=17) and setting the `JAVA11_HOME` and `JAVA17_HOME` environment variables. More to that, since 8.10 release, Gradle has deprecated the usage of the any JDKs below JDK-16. For smooth development experience, the recommendation is to install at least [JDK 17](https://adoptium.net/temurin/releases/?version=17) or [JDK 21](https://adoptium.net/temurin/releases/?version=21). If you still want to build with JDK-11 only, please add `-Dorg.gradle.warning.mode=none` when invoking any Gradle build task from command line, for example: -#### JDK 14 - -To run the full suite of tests, download and install [JDK 14](https://jdk.java.net/archive/) and set `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility). - -#### JDK 17 +``` +./gradlew check -Dorg.gradle.warning.mode=none +``` -By default, the test tasks use bundled JDK runtime, configured in [buildSrc/version.properties](buildSrc/version.properties), and set to JDK 17 (LTS). +By default, the test tasks use bundled JDK runtime, configured in version catalog [gradle/libs.versions.toml](gradle/libs.versions.toml), and set to JDK 23 (non-LTS). ``` bundled_jdk_vendor = adoptium -bundled_jdk = 17.0.2+8 +bundled_jdk = 23.0.1+11 ``` #### Custom Runtime JDK From efef67693279dca89b07510bf1f30acf70c21d63 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Fri, 15 Nov 2024 21:08:26 +0800 Subject: [PATCH 332/357] Update version check in yaml test file for allowing composite aggregation to run under a parent filter aggregation (#16652) Signed-off-by: Gao Binlong --- .../test/search.aggregation/230_composite.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 408e3c9fe0f9f..68d2a899d46cb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -512,8 +512,8 @@ setup: --- "Composite aggregation with filtered nested parent": - skip: - version: " - 2.99.99" - reason: fixed in 3.0.0 + version: " - 2.12.99" + reason: fixed in 2.13.0 - do: search: rest_total_hits_as_int: true @@ -582,8 +582,8 @@ setup: --- "Composite aggregation with filtered reverse nested parent": - skip: - version: " - 2.99.99" - reason: fixed in 3.0.0 + version: " - 2.12.99" + reason: fixed in 2.13.0 - do: search: rest_total_hits_as_int: true From 3b4fa0e86868a5aba594f337b760df116362b3fb Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 15 Nov 2024 13:30:52 -0500 Subject: [PATCH 333/357] Bump mockito from 5.14.1 to 5.14.2, objenesis from 3.2 to 3.3 and bytebuddy from 1.15.4 to 1.15.10 (#16655) * Bump mockito from 5.14.1 to 5.14.2 Signed-off-by: Craig Perkins * Update objenesis from 3.2 to 3.3 Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Update bytebuddy from 1.15.4 to 1.15.10 Signed-off-by: Craig Perkins * Update README Signed-off-by: Craig Perkins * Add grant to mockito-core Signed-off-by: Craig Perkins * Add perms Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- CHANGELOG.md | 1 + gradle/libs.versions.toml | 6 +++--- .../org/opensearch/bootstrap/test-framework.policy | 7 +++++++ test/framework/build.gradle | 3 ++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5991d9593e259..41873adf5f2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 9.46 ([#16611](https://github.com/opensearch-project/OpenSearch/pull/16611)) - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) +- Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655)) ### Changed diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6b5c10e071d1e..0695bbcdf067b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -62,9 +62,9 @@ bouncycastle="1.78" randomizedrunner = "2.7.1" junit = "4.13.2" hamcrest = "2.1" -mockito = "5.14.1" -objenesis = "3.2" -bytebuddy = "1.15.4" +mockito = "5.14.2" +objenesis = "3.3" +bytebuddy = "1.15.10" # benchmark dependencies jmh = "1.35" diff --git a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy index 19f8adbe003ca..c62adda511140 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/test-framework.policy @@ -45,6 +45,13 @@ grant codeBase "${codebase.mockito-core}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; permission java.lang.RuntimePermission "getClassLoader"; + + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "net.bytebuddy.createJavaDispatcher"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.utility"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.dynamic.loading"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.type"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.method"; }; grant codeBase "${codebase.objenesis}" { diff --git a/test/framework/build.gradle b/test/framework/build.gradle index c65bf51c6af36..84a536fdf62c8 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -80,7 +80,8 @@ thirdPartyAudit.ignoreMissingClasses( 'org.apache.log4j.Priority', 'org.mockito.internal.creation.bytebuddy.inject.MockMethodDispatcher', 'org.opentest4j.AssertionFailedError', - 'net.bytebuddy.agent.ByteBuddyAgent' + 'net.bytebuddy.agent.ByteBuddyAgent', + 'net.bytebuddy.agent.Installer' ) // TODO - OpenSearch remove this violation. Issue: https://github.com/opensearch-project/OpenSearch/issues/420 thirdPartyAudit.ignoreViolations( From 8e097eedee288cdb9f974a98bca9325565a64b8e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 15 Nov 2024 18:54:10 -0500 Subject: [PATCH 334/357] Update to Netty 4.1.115.Final (#16661) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + gradle/libs.versions.toml | 2 +- modules/transport-netty4/build.gradle | 3 +++ .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.115.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 | 1 + plugins/repository-s3/build.gradle | 3 +++ .../repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.115.Final.jar.sha1 | 1 + .../repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 - .../repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-classes-epoll-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-classes-epoll-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.115.Final.jar.sha1 | 1 + plugins/transport-nio/build.gradle | 3 +++ .../transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.115.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.115.Final.jar.sha1 | 1 + plugins/transport-reactor-netty4/build.gradle | 3 +++ .../licenses/netty-buffer-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.114.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.115.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.114.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.115.Final.jar.sha1 | 1 + 96 files changed, 59 insertions(+), 46 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 delete mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 create mode 100644 plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 41873adf5f2ae..e65dbbdb908eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `lycheeverse/lychee-action` from 2.0.2 to 2.1.0 ([#16610](https://github.com/opensearch-project/OpenSearch/pull/16610)) - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655)) +- Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661)) ### Changed diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0695bbcdf067b..55e338efc1cca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -32,7 +32,7 @@ grpc = "1.68.0" # when updating the JNA version, also update the version in buildSrc/build.gradle jna = "5.13.0" -netty = "4.1.114.Final" +netty = "4.1.115.Final" joda = "2.12.7" # project reactor diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index a8a165df637a2..cdaf8350055f0 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -240,6 +240,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 04a48547adb05..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..fc735d97148ef --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d39b9866939cbbb8ae3a1af5c1df5ddf93656d47 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 deleted file mode 100644 index acbab117e4c15..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42b1159cac3d196f6bdbd528e29f0fab9dbaae06 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..8a617d4d857cf --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +955faadc8f5b1ca41881398043b62ce1a245800c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 deleted file mode 100644 index 891ed0a444dc2..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a01071edffb4812009312b461ce5f160cdec9b75 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..13156914a5fd6 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +5f61f44704a09b373167f1c0b1854bdcd880a8cb \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7df7a05cd7345..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e443fa3fa383f --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +33b0636c6fe36c7f41d9da204aa1ad94ff244ac7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 deleted file mode 100644 index 32022e1b2eaa4..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -08134b298d48eec5ddf2e3674a978b52e4667304 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..4491099ace714 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +1d65f327e8406f80f744060e10135dd5f61a369a \ No newline at end of file diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 22aa151c92003..398611a016ed2 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -583,6 +583,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7120cd93e5c0d..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f442c794e6fe89e6974f058bf393353e01fb927d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aea007d98763e --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +11fea00408ecbd8b8d1f0698d708e37db4a01841 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index c0f0150378434..7132c97864238 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -175,6 +175,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/build.gradle b/plugins/transport-reactor-netty4/build.gradle index 089e57f062a9f..1e76d1a29efc1 100644 --- a/plugins/transport-reactor-netty4/build.gradle +++ b/plugins/transport-reactor-netty4/build.gradle @@ -258,6 +258,9 @@ thirdPartyAudit { 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 deleted file mode 100644 index 8cb83fc367d78..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1d77d15c0b781cd9395a2a956262766fd0c7602 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 deleted file mode 100644 index 1be26fee34d46..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a49dfa2828d64bf756f670e63259115332744cf \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e39392f923f7b --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d326bf3a4c785b272da3db6941779a1bd5448378 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 04a48547adb05..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35798729ba06670fb4fcd02db98d9577e363992d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..fc735d97148ef --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d39b9866939cbbb8ae3a1af5c1df5ddf93656d47 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 deleted file mode 100644 index e683773245716..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbce5a53884275662e68aaad70f88bf7e5d04164 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..ce4bdc323b2a0 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +80f0dece29a2c0269217e8dd1b6db6ff9710781f \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 deleted file mode 100644 index 2c18924e33c62..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19ae07fdf99142a70338f8cda70a3d2edbc8e80a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..f718fc9bf1622 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +0bc474c27c96e3a309da73160fbcfe0bd3aa85bc \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index 74ce939dc6190..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -862712e292b162c8ccaa7847a6a54df8178f77e5 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 deleted file mode 100644 index c431976b6fbd2..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e56fbde4b9aa628eed15a5dbfbeb97877db88146 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..5c58e02209691 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d54dbf68b9d88a98240107758c6b63da5e46e23a \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 deleted file mode 100644 index e8080a5b2acb1..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10b23784b23d6a948930f52ba82874f1291b5873 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..b1d9f1f679966 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +e33b4d476c03975957f5d8d0319d592bf2bc5e96 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 deleted file mode 100644 index 7df7a05cd7345..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fed36ff50059ded641fa5064963c4b4313512f3 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..e443fa3fa383f --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +33b0636c6fe36c7f41d9da204aa1ad94ff244ac7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 deleted file mode 100644 index fb2d518789a18..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e0225a575f487904be8517092cbd74e01913533c \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..aeeda9c282b30 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +39cef77c1a25908ac1abf4960c2e789f0bf70ff9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 deleted file mode 100644 index a80b9e51be74b..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d1171bb99411f282068f49d780cedf8c9adeabfd \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..41996121685f0 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +dc96c67d06cd6b5eb677f2728f27bf2e3d9a7284 \ No newline at end of file From d4d70d886ed5da1b657dca4c1e5af80ce9cd07ef Mon Sep 17 00:00:00 2001 From: Mikhail Khludnev Date: Sat, 16 Nov 2024 02:59:01 +0300 Subject: [PATCH 335/357] Search dv only IP masks (#16628) * search dv only ip masks Signed-off-by: mikhail-khludnev * search dv only ip masks: changes Signed-off-by: Mikhail Khludnev * drop fancy closures Signed-off-by: mikhail-khludnev * drop fancy closures. one more Signed-off-by: mikhail-khludnev * fix unit tests; add some more dvOnly Signed-off-by: mikhail-khludnev * drop skipping dvOnly in 2.x Signed-off-by: mikhail-khludnev * drop redundant brackets Signed-off-by: mikhail-khludnev * extract conditions Signed-off-by: mikhail-khludnev * asserts Signed-off-by: mikhail-khludnev * spotless apply Signed-off-by: mikhail-khludnev * bring back skip before Signed-off-by: mikhail-khludnev * combine asserts Signed-off-by: mikhail-khludnev * inline, copy-paste Signed-off-by: mikhail-khludnev --------- Signed-off-by: mikhail-khludnev Signed-off-by: Mikhail Khludnev Co-authored-by: mikhail-khludnev --- CHANGELOG.md | 1 + .../test/search/340_doc_values_field.yml | 92 ++++++++++++++- .../index/mapper/IpFieldMapper.java | 86 +++++++------- .../index/mapper/IpFieldTypeTests.java | 107 +++++++++++++++++- 4 files changed, 240 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e65dbbdb908eb..00754925ea111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543)) - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544)) +- Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index a133060f07c6f..647aaf2c9088b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -449,6 +449,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -987,6 +1009,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1077,8 +1121,8 @@ "search on fields with only doc_values enabled": - skip: features: [ "headers" ] - version: " - 2.99.99" - reason: "searching with only doc_values was added in 3.0.0" + version: " - 2.18.99" + reason: "searching with only doc_values was finally added in 2.19.0" - do: indices.create: index: test-doc-values @@ -1372,6 +1416,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + ip_field: ["192.168.0.1", "192.168.0.2"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + ip_field: ["192.168.0.1/31", "192.168.0.3"] + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -1516,6 +1582,28 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + ip_field: "192.168.0.1/31" + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + ip_field: "192.168.0.1/24" + + - match: { hits.total: 3 } + - do: search: rest_total_hits_as_int: true diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index db8da8a949d6f..c51cada9f3143 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -64,6 +64,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * A {@link FieldMapper} for ip addresses. @@ -225,9 +226,9 @@ protected Object parseSourceValue(Object value) { @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { failIfNotIndexedAndNoDocValues(); - Query query; + final PointRangeQuery pointQuery; if (value instanceof InetAddress) { - query = InetAddressPoint.newExactQuery(name(), (InetAddress) value); + pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), (InetAddress) value); } else { if (value instanceof BytesRef) { value = ((BytesRef) value).utf8ToString(); @@ -235,32 +236,27 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { String term = value.toString(); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); - query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + pointQuery = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } else { InetAddress address = InetAddresses.forString(term); - query = InetAddressPoint.newExactQuery(name(), address); + pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), address); } } - if (isSearchable() && hasDocValues()) { - String term = value.toString(); - if (term.contains("/")) { - final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); - } - return new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())) + Query dvQuery = null; + if (hasDocValues()) { + dvQuery = SortedSetDocValuesField.newSlowRangeQuery( + name(), + new BytesRef(pointQuery.getLowerPoint()), + new BytesRef(pointQuery.getUpperPoint()), + true, + true ); } - if (hasDocValues()) { - String term = value.toString(); - if (term.contains("/")) { - final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); - } - return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())); + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; } - return query; } @Override @@ -285,36 +281,46 @@ public Query termsQuery(List values, QueryShardContext context) { } addresses[i++] = address; } - return InetAddressPoint.newSetQuery(name(), addresses); + Query dvQuery = null; + if (hasDocValues()) { + List bytesRefs = Arrays.stream(addresses) + .distinct() + .map(InetAddressPoint::encode) + .map(BytesRef::new) + .collect(Collectors.toList()); + dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs); + } + Query pointQuery = null; + if (isSearchable()) { + pointQuery = InetAddressPoint.newSetQuery(name(), addresses); + } + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; + } } @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> { - Query query = InetAddressPoint.newRangeQuery(name(), lower, upper); - if (isSearchable() && hasDocValues()) { - return new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) - ); - } + PointRangeQuery pointQuery = (PointRangeQuery) InetAddressPoint.newRangeQuery(name(), lower, upper); + Query dvQuery = null; if (hasDocValues()) { - return SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), + dvQuery = SortedSetDocValuesField.newSlowRangeQuery( + pointQuery.getField(), + new BytesRef(pointQuery.getLowerPoint()), + new BytesRef(pointQuery.getUpperPoint()), true, true ); } - return query; + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery(pointQuery, dvQuery); + } else { + return isSearchable() ? pointQuery : dvQuery; + } }); } diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index 0a2435553b19e..a5403ef81481f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -50,6 +50,8 @@ import java.net.InetAddress; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.Objects; public class IpFieldTypeTests extends FieldTypeTestCase { @@ -76,7 +78,7 @@ public void testValueForSearch() { } public void testTermQuery() { - MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); + MappedFieldType ft = new IpFieldMapper.IpFieldType("field", true, false, true, null, Collections.emptyMap()); String ip = "2001:db8::2:1"; @@ -104,20 +106,94 @@ public void testTermQuery() { String prefix = ip + "/64"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64); - assertEquals(query, ft.termQuery(prefix, null)); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("2001:db8:0:0:0:0:0:0"), + ipToByteRef("2001:db8:0:0:ffff:ffff:ffff:ffff"), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); ip = "192.168.1.7"; prefix = ip + "/16"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16); - assertEquals(query, ft.termQuery(prefix, null)); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("::ffff:192.168.0.0"), + ipToByteRef("::ffff:192.168.255.255"), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, false, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } + public void testDvOnlyTermQuery() { + IpFieldMapper.IpFieldType dvOnly = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + String ip = "2001:db8::2:1"; + + Query query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + + assertEquals( + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())), + dvOnly.termQuery(ip, null) + ); + + ip = "192.168.1.7"; + query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + assertEquals( + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())), + dvOnly.termQuery(ip, null) + ); + + ip = "2001:db8::2:1"; + String prefix = ip + "/64"; + + assertEquals( + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("2001:db8:0:0:0:0:0:0"), + ipToByteRef("2001:db8:0:0:ffff:ffff:ffff:ffff"), + true, + true + ), + dvOnly.termQuery(prefix, null) + ); + + ip = "192.168.1.7"; + prefix = ip + "/16"; + assertEquals( + SortedSetDocValuesField.newSlowRangeQuery( + "field", + ipToByteRef("::ffff:192.168.0.0"), + ipToByteRef("::ffff:192.168.255.255"), + true, + true + ), + dvOnly.termQuery(prefix, null) + ); + } + + private static BytesRef ipToByteRef(String ipString) { + return new BytesRef(Objects.requireNonNull(InetAddresses.ipStringToBytes(ipString))); + } + public void testTermsQuery() { - MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); + MappedFieldType ft = new IpFieldMapper.IpFieldType("field", true, false, false, null, Collections.emptyMap()); assertEquals( InetAddressPoint.newSetQuery("field", InetAddresses.forString("::2"), InetAddresses.forString("::5")), @@ -139,6 +215,29 @@ public void testTermsQuery() { ); } + public void testDvOnlyTermsQuery() { + MappedFieldType dvOnly = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + + assertEquals( + SortedSetDocValuesField.newSlowSetQuery("field", List.of(ipToByteRef("::2"), ipToByteRef("::5"))), + dvOnly.termsQuery(Arrays.asList(InetAddresses.forString("::2"), InetAddresses.forString("::5")), null) + ); + assertEquals( + SortedSetDocValuesField.newSlowSetQuery("field", List.of(ipToByteRef("::2"), ipToByteRef("::5"))), + dvOnly.termsQuery(Arrays.asList("::2", "::5"), null) + ); + + // if the list includes a prefix query we fallback to a bool query + assertEquals( + new ConstantScoreQuery( + new BooleanQuery.Builder().add(dvOnly.termQuery("::42", null), Occur.SHOULD) + .add(dvOnly.termQuery("::2/16", null), Occur.SHOULD) + .build() + ), + dvOnly.termsQuery(Arrays.asList("::42", "::2/16"), null) + ); + } + public void testRangeQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); Query query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE); From dccd40f8455b3e1bc39a3f3633d9ebced5c108de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:58:01 -0500 Subject: [PATCH 336/357] Bump org.xerial.snappy:snappy-java from 1.1.10.6 to 1.1.10.7 in /test/fixtures/hdfs-fixture (#16665) * Bump org.xerial.snappy:snappy-java in /test/fixtures/hdfs-fixture Bumps [org.xerial.snappy:snappy-java](https://github.com/xerial/snappy-java) from 1.1.10.6 to 1.1.10.7. - [Release notes](https://github.com/xerial/snappy-java/releases) - [Commits](https://github.com/xerial/snappy-java/compare/v1.1.10.6...v1.1.10.7) --- updated-dependencies: - dependency-name: org.xerial.snappy:snappy-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00754925ea111..2e9ecea13af60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `me.champeau.gradle.japicmp` from 0.4.4 to 0.4.5 ([#16614](https://github.com/opensearch-project/OpenSearch/pull/16614)) - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655)) - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661)) +- Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) ### Changed diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index bfa6c65ee6ec8..55e27c7d6169b 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -89,5 +89,5 @@ dependencies { exclude group: "com.squareup.okio" } runtimeOnly "com.squareup.okio:okio:3.9.1" - runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.6" + runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.7" } From f105e4eb2ede1556b5dd3c743bea1ab9686ebccf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:37:58 -0500 Subject: [PATCH 337/357] Bump com.gradle.develocity from 3.18.1 to 3.18.2 (#16669) Bumps com.gradle.develocity from 3.18.1 to 3.18.2. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 8412d198a2a29..035fe69eda7e9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.develocity" version "3.18.1" + id "com.gradle.develocity" version "3.18.2" } ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') From ac443003bad13aeb6373f80439e154dcf25023a2 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:40:39 +0530 Subject: [PATCH 338/357] Fix stale cluster state custom file deletion (#16670) Signed-off-by: Sooraj Sinha --- CHANGELOG.md | 1 + .../gateway/remote/RemoteClusterStateCleanupManager.java | 2 +- .../remote/RemoteClusterStateCleanupManagerTests.java | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e9ecea13af60..1c07a9362e3e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483)) - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544)) - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628)) +- Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670)) ### Security diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index d9bd9669f138c..559b7422312c0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -306,7 +306,7 @@ void deleteClusterMetadata( staleEphemeralAttributePaths.add(clusterMetadataManifest.getHashesOfConsistentSettings().getUploadedFilename()); } if (clusterMetadataManifest.getClusterStateCustomMap() != null) { - clusterMetadataManifest.getCustomMetadataMap() + clusterMetadataManifest.getClusterStateCustomMap() .values() .stream() .filter(u -> !filesToKeep.contains(u.getUploadedFilename())) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 8e114c9a26534..753e791ca915c 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -42,6 +42,7 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -200,6 +201,7 @@ public void testDeleteClusterMetadata() throws IOException { .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID(ClusterState.UNKNOWN_UUID) + .clusterStateCustomMetadataMap(Map.of("snapshots", new UploadedMetadataAttribute("snapshots", "snapshot_file1"))) .committed(true) .build(); ClusterMetadataManifest manifest2 = ClusterMetadataManifest.builder(manifest1) @@ -209,10 +211,12 @@ public void testDeleteClusterMetadata() throws IOException { .coordinationMetadata(coordinationMetadata) .templatesMetadata(templateMetadata) .settingMetadata(settingMetadata) + .clusterStateCustomMetadataMap(Map.of("restore", new UploadedMetadataAttribute("restore", "restore_file1"))) .build(); ClusterMetadataManifest manifest3 = ClusterMetadataManifest.builder(manifest2) .indices(List.of(index1UpdatedMetadata, index2Metadata)) .settingMetadata(settingMetadataUpdated) + .clusterStateCustomMetadataMap(Map.of()) .build(); UploadedIndexMetadata index3Metadata = new UploadedIndexMetadata("index3", "indexUUID3", "index_metadata3__2"); @@ -286,6 +290,7 @@ public void testDeleteClusterMetadata() throws IOException { ) ); verify(container).deleteBlobsIgnoringIfNotExists(List.of(getFormattedIndexFileName(index1Metadata.getUploadedFilePath()))); + verify(container).deleteBlobsIgnoringIfNotExists(List.of("restore_file1", "snapshot_file1")); Set staleManifest = new HashSet<>(); inactiveBlobs.forEach( blob -> staleManifest.add( From 9b8b9fddb1902ca73fdc11766c43433dc38e592f Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 19 Nov 2024 08:46:46 -0500 Subject: [PATCH 339/357] Update Gradle to 8.11 (#16386) Signed-off-by: Andriy Redko --- build.gradle | 20 ++++++++++++------- .../test/TestWithDependenciesPlugin.groovy | 9 ++++++--- .../org/opensearch/gradle/LoggedExec.java | 11 ++++++++-- .../org/opensearch/gradle/PublishPlugin.java | 2 +- .../gradle/precommit/ThirdPartyAuditTask.java | 14 +++++++++++-- gradle/missing-javadoc.gradle | 9 ++++++++- gradle/wrapper/gradle-wrapper.properties | 4 ++-- libs/build.gradle | 2 +- modules/lang-painless/build.gradle | 8 +++++++- 9 files changed, 59 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 55b31ca816214..f720b46bec143 100644 --- a/build.gradle +++ b/build.gradle @@ -162,14 +162,20 @@ Map buildMetadataMap = buildMetadataValue.tokenize(';').collectE return [key, value] } - /** - * Using 'git' command line (if available), tries to fetch the commit date of the current revision - * @return commit date of the current revision or 0 if it is not available - */ +// See please https://docs.gradle.org/8.11/userguide/service_injection.html#execoperations +interface InjectedExecOps { + @Inject ExecOperations getExecOps() +} + +/** + * Using 'git' command line (if available), tries to fetch the commit date of the current revision + * @return commit date of the current revision or 0 if it is not available + */ long gitRevisionDate = { + def execOps = project.objects.newInstance(InjectedExecOps) // Try to get last commit date as Unix timestamp try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) { - ExecResult result = project.exec(spec -> { + ExecResult result = execOps.execOps.exec(spec -> { spec.setIgnoreExitValue(true); spec.setStandardOutput(stdout); spec.commandLine("git", "log", "-1", "--format=%ct"); @@ -362,7 +368,7 @@ allprojects { if ((dep instanceof ProjectDependency) == false) { return } - Project upstreamProject = dep.dependencyProject + Project upstreamProject = project.project(dep.path) if (upstreamProject == null) { return } @@ -438,7 +444,7 @@ gradle.projectsEvaluated { configurations.matching { it.canBeResolved }.all { Configuration configuration -> dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> - Project upstreamProject = dep.dependencyProject + Project upstreamProject = project.project(dep.path) if (upstreamProject != null) { if (project.path == upstreamProject.path) { // TODO: distribution integ tests depend on themselves (!), fix that diff --git a/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy b/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy index e84493d442f39..30430296d6383 100644 --- a/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy +++ b/buildSrc/src/main/groovy/org/opensearch/gradle/test/TestWithDependenciesPlugin.groovy @@ -56,9 +56,12 @@ class TestWithDependenciesPlugin implements Plugin { project.configurations.testImplementation.dependencies.all { Dependency dep -> // this closure is run every time a compile dependency is added - if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) { - project.gradle.projectsEvaluated { - addPluginResources(project, dep.dependencyProject) + if (dep instanceof ProjectDependency) { + Project dependencyProject = project.project(((ProjectDependency)dep).path) + if (dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) { + project.gradle.projectsEvaluated { + addPluginResources(project, dependencyProject) + } } } } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java b/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java index 1a78a7dbb2d10..4c62f4a6b4ee8 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java @@ -71,6 +71,11 @@ public class LoggedExec extends Exec implements FileSystemOperationsAware { private Consumer outputLogger; private FileSystemOperations fileSystemOperations; + interface InjectedExecOps { + @Inject + ExecOperations getExecOps(); + } + @Inject public LoggedExec(FileSystemOperations fileSystemOperations) { this.fileSystemOperations = fileSystemOperations; @@ -133,7 +138,8 @@ public void setSpoolOutput(boolean spoolOutput) { } public static ExecResult exec(Project project, Action action) { - return genericExec(project::exec, action); + final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class); + return exec(execOps.getExecOps(), action); } public static ExecResult exec(ExecOperations execOperations, Action action) { @@ -141,7 +147,8 @@ public static ExecResult exec(ExecOperations execOperations, Action ac } public static ExecResult javaexec(Project project, Action action) { - return genericExec(project::javaexec, action); + final InjectedExecOps execOps = project.getObjects().newInstance(InjectedExecOps.class); + return genericExec(execOps.getExecOps()::javaexec, action); } /** Returns JVM arguments suitable for a short-lived forked task */ diff --git a/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java index 7ec21bba18c64..4db4faa8cbe9f 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java @@ -121,7 +121,7 @@ public String call() throws Exception { Node dependencyNode = dependenciesNode.appendNode("dependency"); dependencyNode.appendNode("groupId", dependency.getGroup()); ProjectDependency projectDependency = (ProjectDependency) dependency; - String artifactId = getArchivesBaseName(projectDependency.getDependencyProject()); + String artifactId = getArchivesBaseName(project.project(projectDependency.getPath())); dependencyNode.appendNode("artifactId", artifactId); dependencyNode.appendNode("version", dependency.getVersion()); dependencyNode.appendNode("scope", "compile"); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java index f7bb708933803..a74781ac44720 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java @@ -60,8 +60,11 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.SkipWhenEmpty; import org.gradle.api.tasks.TaskAction; +import org.gradle.process.ExecOperations; import org.gradle.process.ExecResult; +import javax.inject.Inject; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -108,6 +111,11 @@ public class ThirdPartyAuditTask extends DefaultTask { public boolean jarHellEnabled = true; + interface InjectedExecOps { + @Inject + ExecOperations getExecOps(); + } + @Input public Property getTargetCompatibility() { return targetCompatibility; @@ -357,7 +365,8 @@ private String formatClassList(Set classList) { private String runForbiddenAPIsCli() throws IOException { ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); - ExecResult result = getProject().javaexec(spec -> { + InjectedExecOps execOps = getProject().getObjects().newInstance(InjectedExecOps.class); + ExecResult result = execOps.getExecOps().javaexec(spec -> { if (javaHome != null) { spec.setExecutable(javaHome + "/bin/java"); } @@ -391,7 +400,8 @@ private String runForbiddenAPIsCli() throws IOException { private Set runJdkJarHellCheck() throws IOException { ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); - ExecResult execResult = getProject().javaexec(spec -> { + InjectedExecOps execOps = getProject().getObjects().newInstance(InjectedExecOps.class); + ExecResult execResult = execOps.getExecOps().javaexec(spec -> { spec.classpath( jdkJarHellClasspath, getRuntimeConfiguration(), diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle index 26898673bf608..751da941d25dd 100644 --- a/gradle/missing-javadoc.gradle +++ b/gradle/missing-javadoc.gradle @@ -8,6 +8,7 @@ import javax.annotation.Nullable +import javax.inject.Inject import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.internal.jvm.Jvm @@ -227,6 +228,11 @@ class MissingJavadocTask extends DefaultTask { @PathSensitive(PathSensitivity.RELATIVE) def taskResources + // See please https://docs.gradle.org/8.11/userguide/service_injection.html#execoperations + interface InjectedExecOps { + @Inject ExecOperations getExecOps() + } + /** Utility method to recursively collect all tasks with same name like this one that we depend on */ private Set findRenderTasksInDependencies() { Set found = [] @@ -317,11 +323,12 @@ class MissingJavadocTask extends DefaultTask { } }() + def execOps = project.objects.newInstance(InjectedExecOps) def outputFile = project.file("${getTemporaryDir()}/javadoc-output.txt") def result outputFile.withOutputStream { output -> - result = project.exec { + result = execOps.execOps.exec { executable javadocCmd // we want to capture both stdout and stderr to the same diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e312a2da77d94..3bfe9cc6bd3c2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6 +distributionSha256Sum=73d2d553933194d8eefed0a291acbe45392ca3572ba13834cbbf373da375276d diff --git a/libs/build.gradle b/libs/build.gradle index 39d2737966b6d..c0fcc1ff2b977 100644 --- a/libs/build.gradle +++ b/libs/build.gradle @@ -42,7 +42,7 @@ subprojects { project.afterEvaluate { configurations.all { Configuration conf -> dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> - Project depProject = dep.dependencyProject + Project depProject = project.project(dep.path) if (depProject != null && (false == depProject.path.equals(':libs:opensearch-core') && false == depProject.path.equals(':libs:opensearch-common')) diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 7075901979e3b..ffb1fe6117c06 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -115,11 +115,17 @@ testClusters { } } +interface InjectedExecOps { + @Inject ExecOperations getExecOps() +} + + tasks.register("generateContextDoc", DefaultTestClustersTask) { dependsOn sourceSets.doc.runtimeClasspath useCluster testClusters.generateContextCluster doFirst { - project.javaexec { + def execOps = project.objects.newInstance(InjectedExecOps) + execOps.execOps.javaexec { mainClass = 'org.opensearch.painless.ContextDocGenerator' classpath = sourceSets.doc.runtimeClasspath systemProperty "cluster.uri", "${-> testClusters.generateContextCluster.singleNode().getAllHttpSocketURI().get(0)}" From 1d8568eede1bae3b2df3881982adb2d5a247b7b8 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 19 Nov 2024 10:26:00 -0500 Subject: [PATCH 340/357] Update Gradle version for Detect Breaking Changes Github action (#16685) --- .github/workflows/detect-breaking-change.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/detect-breaking-change.yml b/.github/workflows/detect-breaking-change.yml index e5d3fddbd36f5..7b223268c48bd 100644 --- a/.github/workflows/detect-breaking-change.yml +++ b/.github/workflows/detect-breaking-change.yml @@ -17,7 +17,7 @@ jobs: with: cache-disabled: true arguments: japicmp - gradle-version: 8.7 + gradle-version: 8.11 build-root-directory: server - if: failure() run: cat server/build/reports/java-compatibility/report.txt From 2ac64a6262973383d71f3e3ee037e4d267c3b592 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Tue, 19 Nov 2024 09:15:09 -0800 Subject: [PATCH 341/357] Changing request cache size > 0 setting to int threshold (#16570) Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi --- .../indices/IndicesRequestCacheIT.java | 20 +++++++++++++++---- .../common/settings/ClusterSettings.java | 2 +- .../indices/IndicesRequestCache.java | 16 +++++++++------ .../opensearch/indices/IndicesService.java | 17 ++++++++-------- .../indices/IndicesServiceTests.java | 13 ++++++------ 5 files changed, 42 insertions(+), 26 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index bab085bf265af..a16d2065598ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -90,7 +90,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING; -import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING; +import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; @@ -582,21 +582,33 @@ public void testCanCache() throws Exception { assertThat(r4.getHits().getTotalHits().value, equalTo(7L)); assertCacheState(client, index, 0, 4); - // If size > 0 we should cache if this is enabled via cluster setting + // Update max cacheable size for request cache from default value of 0 ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + int maxCacheableSize = 5; updateSettingsRequest.persistentSettings( - Settings.builder().put(INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING.getKey(), true) + Settings.builder().put(INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING.getKey(), maxCacheableSize) ); assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + // Sizes <= the cluster setting value should be cached final SearchResponse r7 = client.prepareSearch(index) .setSearchType(SearchType.QUERY_THEN_FETCH) - .setSize(1) + .setSize(maxCacheableSize) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-22").lte("2016-03-26")) .get(); OpenSearchAssertions.assertAllSuccessful(r7); assertThat(r7.getHits().getTotalHits().value, equalTo(5L)); assertCacheState(client, index, 0, 6); + + // Sizes > the cluster setting value should not be cached + final SearchResponse r8 = client.prepareSearch(index) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setSize(maxCacheableSize + 1) + .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-22").lte("2016-03-26")) + .get(); + OpenSearchAssertions.assertAllSuccessful(r8); + assertThat(r8.getHits().getTotalHits().value, equalTo(5L)); + assertCacheState(client, index, 0, 6); } public void testCacheWithFilteredAlias() throws InterruptedException { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index c836984655ad1..04a19e32c4ebc 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -519,7 +519,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesRequestCache.INDICES_CACHE_QUERY_EXPIRE, IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING, IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING, - IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING, + IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING, HunspellService.HUNSPELL_LAZY_LOAD, HunspellService.HUNSPELL_IGNORE_CASE, HunspellService.HUNSPELL_DICTIONARY_OPTIONS, diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index 4dde4445cd483..3d158cb60a208 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -148,13 +148,17 @@ public final class IndicesRequestCache implements RemovalListener 0 queries. - * If enabled, fundamentally non-cacheable queries like DFS queries, queries using the `now` keyword, and - * scroll requests are still not cached. + * Sets the maximum size of a query which is allowed in the request cache. + * This refers to the number of documents returned, not the size in bytes. + * Default value of 0 only allows size == 0 queries, matching earlier behavior. + * Fundamentally non-cacheable queries like DFS queries, queries using the `now` keyword, and + * scroll requests are never cached, regardless of this setting. */ - public static final Setting INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING = Setting.boolSetting( - "indices.requests.cache.enable_for_all_requests", - false, + public static final Setting INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING = Setting.intSetting( + "indices.requests.cache.maximum_cacheable_size", + 0, + 0, + 10_000, Property.NodeScope, Property.Dynamic ); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 1a4c9067939a9..b9bad5527e3f4 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -205,7 +205,7 @@ import static org.opensearch.index.IndexService.IndexCreationContext.CREATE_INDEX; import static org.opensearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION; import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; -import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING; +import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent; import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -361,7 +361,7 @@ public class IndicesService extends AbstractLifecycleComponent private final FileCache fileCache; private final CompositeIndexSettings compositeIndexSettings; private final Consumer replicator; - private volatile boolean requestCachingEnabledForAllQueries; + private volatile int maxSizeInRequestCache; @Override protected void doStart() { @@ -509,9 +509,9 @@ protected void closeInternal() { this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; this.replicator = replicator; - this.requestCachingEnabledForAllQueries = INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING.get(clusterService.getSettings()); + this.maxSizeInRequestCache = INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings() - .addSettingsUpdateConsumer(INDICES_REQUEST_CACHE_ENABLE_FOR_ALL_REQUESTS_SETTING, this::setRequestCachingEnabledForAllQueries); + .addSettingsUpdateConsumer(INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING, this::setMaxSizeInRequestCache); } public IndicesService( @@ -1752,10 +1752,9 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) { // if not explicitly set in the request, use the index setting, if not, use the request if (request.requestCache() == null) { if (settings.getValue(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING) == false - || (context.size() > 0 && !requestCachingEnabledForAllQueries)) { + || (context.size() > maxSizeInRequestCache)) { // If no request cache query parameter and shard request cache - // is enabled in settings don't cache for requests with size > 0 - // unless this is enabled via cluster setting + // is enabled in settings, use cluster setting to check the maximum size allowed in the cache return false; } } else if (request.requestCache() == false) { @@ -2125,7 +2124,7 @@ public CompositeIndexSettings getCompositeIndexSettings() { } // Package-private for testing - void setRequestCachingEnabledForAllQueries(Boolean requestCachingEnabledForAllQueries) { - this.requestCachingEnabledForAllQueries = requestCachingEnabledForAllQueries; + void setMaxSizeInRequestCache(Integer maxSizeInRequestCache) { + this.maxSizeInRequestCache = maxSizeInRequestCache; } } diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java index d2250702b48fd..9c717c796daae 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java @@ -652,9 +652,7 @@ public void testDirectoryReaderWithoutDelegatingCacheHelperNotCacheable() throws } public void testCanCacheSizeNonzero() { - // Size == 0 requests should always be cacheable (if they pass the other checks). - // Size > 0 requests should only be cacheable if ALLOW_SIZE_NONZERO_SETTING is true. - + // Requests should only be cached if their size is <= INDICES_REQUEST_CACHE_MAX_SIZE_TO_CACHE_SETTING. final IndexService indexService = createIndex("test"); ShardSearchRequest request = mock(ShardSearchRequest.class); when(request.requestCache()).thenReturn(null); @@ -662,7 +660,7 @@ public void testCanCacheSizeNonzero() { TestSearchContext sizeZeroContext = getTestContext(indexService, 0); TestSearchContext sizeNonzeroContext = getTestContext(indexService, 10); - // Test for an IndicesService with the default setting value of false + // Test for an IndicesService with the default setting value of 0 IndicesService indicesService = getIndicesService(); DelegatingCacheHelper cacheHelper = mock(DelegatingCacheHelper.class); Map expectedResultMap = Map.of(sizeZeroContext, true, sizeNonzeroContext, false); @@ -673,8 +671,11 @@ public void testCanCacheSizeNonzero() { assertEquals(entry.getValue(), indicesService.canCache(request, context)); } // Simulate the cluster setting update by manually calling setCanCacheSizeNonzeroRequests - indicesService.setRequestCachingEnabledForAllQueries(true); - expectedResultMap = Map.of(sizeZeroContext, true, sizeNonzeroContext, true); + int maxCacheableSize = 40; + indicesService.setMaxSizeInRequestCache(maxCacheableSize); + TestSearchContext sizeEqualsThresholdContext = getTestContext(indexService, maxCacheableSize); + TestSearchContext sizeAboveThresholdContext = getTestContext(indexService, maxCacheableSize + 5); + expectedResultMap = Map.of(sizeZeroContext, true, sizeEqualsThresholdContext, true, sizeAboveThresholdContext, false); for (Map.Entry entry : expectedResultMap.entrySet()) { TestSearchContext context = entry.getKey(); From b17c295c4bd86e81122ad60d77f2b3704b1b9c91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:43:56 -0500 Subject: [PATCH 342/357] Bump codecov/codecov-action from 4 to 5 (#16667) * Bump codecov/codecov-action from 4 to 5 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4 to 5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- .github/workflows/gradle-check.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 1421eeb7f7576..577ab0c79535b 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -115,7 +115,7 @@ jobs: - name: Upload Coverage Report if: success() - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./codeCoverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c07a9362e3e7..45ea88bcfdea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `mockito` from 5.14.1 to 5.14.2, `objenesis` from 3.2 to 3.3 and `bytebuddy` from 1.15.4 to 1.15.10 ([#16655](https://github.com/opensearch-project/OpenSearch/pull/16655)) - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661)) - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) +- Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667)) ### Changed From 05513dfd94f790eb89a52aa5c42452928e9e0a12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:10:08 -0500 Subject: [PATCH 343/357] Bump com.google.apis:google-api-services-compute from v1-rev20241021-2.0.0 to v1-rev20241105-2.0.0 in /plugins/discovery-gce (#16613) * Bump com.google.apis:google-api-services-compute Bumps com.google.apis:google-api-services-compute from v1-rev20241021-2.0.0 to v1-rev20241105-2.0.0. --- updated-dependencies: - dependency-name: com.google.apis:google-api-services-compute dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- plugins/discovery-gce/build.gradle | 2 +- .../google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 | 1 - .../google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ea88bcfdea3..aad23b59b40f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) - Bump `google-auth-library-oauth2-http` from 1.7.0 to 1.29.0 in /plugins/repository-gcs ([#16520](https://github.com/opensearch-project/OpenSearch/pull/16520)) - Bump `com.azure:azure-storage-common` from 12.25.1 to 12.27.1 ([#16521](https://github.com/opensearch-project/OpenSearch/pull/16521)) -- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241021-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548)) +- Bump `com.google.apis:google-api-services-compute` from v1-rev20240407-2.0.0 to v1-rev20241105-2.0.0 ([#16502](https://github.com/opensearch-project/OpenSearch/pull/16502), [#16548](https://github.com/opensearch-project/OpenSearch/pull/16548), [#16613](https://github.com/opensearch-project/OpenSearch/pull/16613)) - Bump `com.azure:azure-storage-blob` from 12.23.0 to 12.28.1 ([#16501](https://github.com/opensearch-project/OpenSearch/pull/16501)) - Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.0 to 3.4.1 ([#16550](https://github.com/opensearch-project/OpenSearch/pull/16550)) - Bump `org.apache.xmlbeans:xmlbeans` from 5.2.1 to 5.2.2 ([#16612](https://github.com/opensearch-project/OpenSearch/pull/16612)) diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 5f4670357f927..3214db2074198 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -18,7 +18,7 @@ opensearchplugin { } dependencies { - api "com.google.apis:google-api-services-compute:v1-rev20241021-2.0.0" + api "com.google.apis:google-api-services-compute:v1-rev20241105-2.0.0" api "com.google.api-client:google-api-client:1.35.2" api "com.google.oauth-client:google-oauth-client:1.36.0" api "com.google.http-client:google-http-client:${versions.google_http_client}" diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 deleted file mode 100644 index 309d10035f35a..0000000000000 --- a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241021-2.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cc3bd864ec5ac819699ea24a64109bfda42cb55c \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..d035ba3331e8a --- /dev/null +++ b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20241105-2.0.0.jar.sha1 @@ -0,0 +1 @@ +9a82357e628b39e189c575094b384b442bff4f8e \ No newline at end of file From b1a7743b83913ebe0c40744131c5996608e324a0 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 21 Nov 2024 15:51:30 +0530 Subject: [PATCH 344/357] Separating remote download and publication stats (#16682) * Separating remote download and publication stats Signed-off-by: Himshikha Gupta --- CHANGELOG.md | 1 + .../PublicationTransportHandler.java | 4 +- .../remote/RemoteClusterStateService.java | 322 ++++++++++-------- .../gateway/remote/RemoteDownloadStats.java | 11 + .../remote/RemotePersistenceStats.java | 8 + .../PublicationTransportHandlerTests.java | 37 +- .../RemoteClusterStateServiceTests.java | 12 + 7 files changed, 228 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad23b59b40f4..9cfcd4e6dfbd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489)) - Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271)) - Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)). +- Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/)) ### Dependencies - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index c4cb484cda693..7275d72f2db9f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -298,9 +298,9 @@ PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest } } catch (Exception e) { if (applyFullState) { - remoteClusterStateService.fullDownloadFailed(); + remoteClusterStateService.fullIncomingPublicationFailed(); } else { - remoteClusterStateService.diffDownloadFailed(); + remoteClusterStateService.diffIncomingPublicationFailed(); } throw e; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e4f4bae9bef7c..c5fc6d5cae6a7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1470,173 +1470,191 @@ public ClusterState getClusterStateForManifest( String localNodeId, boolean includeEphemeral ) throws IOException { - ClusterState stateFromCache = remoteClusterStateCache.getState(clusterName, manifest); - if (stateFromCache != null) { - return stateFromCache; - } + try { + ClusterState stateFromCache = remoteClusterStateCache.getState(clusterName, manifest); + if (stateFromCache != null) { + return stateFromCache; + } - final ClusterState clusterState; - final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); - if (manifest.onOrAfterCodecVersion(CODEC_V2)) { - clusterState = readClusterStateInParallel( - ClusterState.builder(new ClusterName(clusterName)).build(), - manifest, - manifest.getClusterUUID(), - localNodeId, - manifest.getIndices(), - manifest.getCustomMetadataMap(), - manifest.getCoordinationMetadata() != null, - manifest.getSettingsMetadata() != null, - includeEphemeral && manifest.getTransientSettingsMetadata() != null, - manifest.getTemplatesMetadata() != null, - includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, - includeEphemeral && manifest.getClusterBlocksMetadata() != null, - includeEphemeral ? manifest.getIndicesRouting() : emptyList(), - includeEphemeral && manifest.getHashesOfConsistentSettings() != null, - includeEphemeral ? manifest.getClusterStateCustomMap() : emptyMap(), - false, - includeEphemeral - ); + final ClusterState clusterState; + final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); + if (manifest.onOrAfterCodecVersion(CODEC_V2)) { + clusterState = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + includeEphemeral && manifest.getTransientSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, + includeEphemeral && manifest.getClusterBlocksMetadata() != null, + includeEphemeral ? manifest.getIndicesRouting() : emptyList(), + includeEphemeral && manifest.getHashesOfConsistentSettings() != null, + includeEphemeral ? manifest.getClusterStateCustomMap() : emptyMap(), + false, + includeEphemeral + ); - if (includeEphemeral - && !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) - && manifest.getClusterStateChecksum() != null) { - validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); + if (includeEphemeral + && !remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + && manifest.getClusterStateChecksum() != null) { + validateClusterStateFromChecksum(manifest, clusterState, clusterName, localNodeId, true); + } + } else { + ClusterState state = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + // for manifest codec V1, we don't have the following objects to read, so not passing anything + emptyMap(), + false, + false, + false, + false, + false, + false, + emptyList(), + false, + emptyMap(), + false, + false + ); + Metadata.Builder mb = Metadata.builder(remoteGlobalMetadataManager.getGlobalMetadata(manifest.getClusterUUID(), manifest)); + mb.indices(state.metadata().indices()); + clusterState = ClusterState.builder(state).metadata(mb).build(); } - } else { - ClusterState state = readClusterStateInParallel( - ClusterState.builder(new ClusterName(clusterName)).build(), - manifest, - manifest.getClusterUUID(), - localNodeId, - manifest.getIndices(), - // for manifest codec V1, we don't have the following objects to read, so not passing anything - emptyMap(), - false, - false, - false, - false, - false, - false, - emptyList(), - false, - emptyMap(), - false, - false - ); - Metadata.Builder mb = Metadata.builder(remoteGlobalMetadataManager.getGlobalMetadata(manifest.getClusterUUID(), manifest)); - mb.indices(state.metadata().indices()); - clusterState = ClusterState.builder(state).metadata(mb).build(); - } - final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); - remoteStateStats.stateFullDownloadSucceeded(); - remoteStateStats.stateFullDownloadTook(durationMillis); - if (includeEphemeral) { - // cache only if the entire cluster-state is present - remoteClusterStateCache.putState(clusterState); + final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); + remoteStateStats.stateFullDownloadSucceeded(); + remoteStateStats.stateFullDownloadTook(durationMillis); + if (includeEphemeral) { + // cache only if the entire cluster-state is present + remoteClusterStateCache.putState(clusterState); + } + return clusterState; + } catch (Exception e) { + logger.error("Failure in downloading full cluster state. ", e); + remoteStateStats.stateFullDownloadFailed(); + throw e; } - return clusterState; } public ClusterState getClusterStateUsingDiff(ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) { - assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; - final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); - ClusterStateDiffManifest diff = manifest.getDiffManifest(); - boolean includeEphemeral = true; - - List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { - Optional uploadedIndexMetadataOptional = manifest.getIndices() - .stream() - .filter(idx2 -> idx2.getIndexName().equals(idx)) - .findFirst(); - assert uploadedIndexMetadataOptional.isPresent() == true; - return uploadedIndexMetadataOptional.get(); - }).collect(Collectors.toList()); - - Map updatedCustomMetadata = new HashMap<>(); - if (diff.getCustomMetadataUpdated() != null) { - for (String customType : diff.getCustomMetadataUpdated()) { - updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); + try { + assert manifest.getDiffManifest() != null : "Diff manifest null which is required for downloading cluster state"; + final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); + ClusterStateDiffManifest diff = manifest.getDiffManifest(); + boolean includeEphemeral = true; + + List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { + Optional uploadedIndexMetadataOptional = manifest.getIndices() + .stream() + .filter(idx2 -> idx2.getIndexName().equals(idx)) + .findFirst(); + assert uploadedIndexMetadataOptional.isPresent() == true; + return uploadedIndexMetadataOptional.get(); + }).collect(Collectors.toList()); + + Map updatedCustomMetadata = new HashMap<>(); + if (diff.getCustomMetadataUpdated() != null) { + for (String customType : diff.getCustomMetadataUpdated()) { + updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); + } } - } - Map updatedClusterStateCustom = new HashMap<>(); - if (diff.getClusterStateCustomUpdated() != null) { - for (String customType : diff.getClusterStateCustomUpdated()) { - updatedClusterStateCustom.put(customType, manifest.getClusterStateCustomMap().get(customType)); + Map updatedClusterStateCustom = new HashMap<>(); + if (diff.getClusterStateCustomUpdated() != null) { + for (String customType : diff.getClusterStateCustomUpdated()) { + updatedClusterStateCustom.put(customType, manifest.getClusterStateCustomMap().get(customType)); + } + } + + List updatedIndexRouting = new ArrayList<>(); + if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { + updatedIndexRouting.addAll( + remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata( + diff.getIndicesRoutingUpdated(), + manifest.getIndicesRouting() + ) + ); } - } - List updatedIndexRouting = new ArrayList<>(); - if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { - updatedIndexRouting.addAll( - remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), manifest.getIndicesRouting()) + ClusterState updatedClusterState = readClusterStateInParallel( + previousState, + manifest, + manifest.getClusterUUID(), + localNodeId, + updatedIndices, + updatedCustomMetadata, + diff.isCoordinationMetadataUpdated(), + diff.isSettingsMetadataUpdated(), + diff.isTransientSettingsMetadataUpdated(), + diff.isTemplatesMetadataUpdated(), + diff.isDiscoveryNodesUpdated(), + diff.isClusterBlocksUpdated(), + updatedIndexRouting, + diff.isHashesOfConsistentSettingsUpdated(), + updatedClusterStateCustom, + manifest.getDiffManifest() != null + && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null + && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), + includeEphemeral ); - } + ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); + Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); + // remove the deleted indices from the metadata + for (String index : diff.getIndicesDeleted()) { + metadataBuilder.remove(index); + } + // remove the deleted metadata customs from the metadata + if (diff.getCustomMetadataDeleted() != null) { + for (String customType : diff.getCustomMetadataDeleted()) { + metadataBuilder.removeCustom(customType); + } + } - ClusterState updatedClusterState = readClusterStateInParallel( - previousState, - manifest, - manifest.getClusterUUID(), - localNodeId, - updatedIndices, - updatedCustomMetadata, - diff.isCoordinationMetadataUpdated(), - diff.isSettingsMetadataUpdated(), - diff.isTransientSettingsMetadataUpdated(), - diff.isTemplatesMetadataUpdated(), - diff.isDiscoveryNodesUpdated(), - diff.isClusterBlocksUpdated(), - updatedIndexRouting, - diff.isHashesOfConsistentSettingsUpdated(), - updatedClusterStateCustom, - manifest.getDiffManifest() != null - && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null - && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), - includeEphemeral - ); - ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); - Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); - // remove the deleted indices from the metadata - for (String index : diff.getIndicesDeleted()) { - metadataBuilder.remove(index); - } - // remove the deleted metadata customs from the metadata - if (diff.getCustomMetadataDeleted() != null) { - for (String customType : diff.getCustomMetadataDeleted()) { - metadataBuilder.removeCustom(customType); + // remove the deleted cluster state customs from the metadata + if (diff.getClusterStateCustomDeleted() != null) { + for (String customType : diff.getClusterStateCustomDeleted()) { + clusterStateBuilder.removeCustom(customType); + } } - } - // remove the deleted cluster state customs from the metadata - if (diff.getClusterStateCustomDeleted() != null) { - for (String customType : diff.getClusterStateCustomDeleted()) { - clusterStateBuilder.removeCustom(customType); + HashMap indexRoutingTables = new HashMap<>( + updatedClusterState.getRoutingTable().getIndicesRouting() + ); + if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { + for (String indexName : diff.getIndicesRoutingDeleted()) { + indexRoutingTables.remove(indexName); + } } - } - HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); - if (manifest.getCodecVersion() == CODEC_V2 || manifest.getCodecVersion() == CODEC_V3) { - for (String indexName : diff.getIndicesRoutingDeleted()) { - indexRoutingTables.remove(indexName); + ClusterState clusterState = clusterStateBuilder.stateUUID(manifest.getStateUUID()) + .version(manifest.getStateVersion()) + .metadata(metadataBuilder) + .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) + .build(); + if (!remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) + && manifest.getClusterStateChecksum() != null) { + validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); } - } + final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); + remoteStateStats.stateDiffDownloadSucceeded(); + remoteStateStats.stateDiffDownloadTook(durationMillis); - ClusterState clusterState = clusterStateBuilder.stateUUID(manifest.getStateUUID()) - .version(manifest.getStateVersion()) - .metadata(metadataBuilder) - .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables)) - .build(); - if (!remoteClusterStateValidationMode.equals(RemoteClusterStateValidationMode.NONE) && manifest.getClusterStateChecksum() != null) { - validateClusterStateFromChecksum(manifest, clusterState, previousState.getClusterName().value(), localNodeId, false); + assert includeEphemeral == true; + // newState includes all the fields of cluster-state (includeEphemeral=true always) + remoteClusterStateCache.putState(clusterState); + return clusterState; + } catch (Exception e) { + logger.error("Failure in downloading diff cluster state. ", e); + remoteStateStats.stateDiffDownloadFailed(); + throw e; } - final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); - remoteStateStats.stateDiffDownloadSucceeded(); - remoteStateStats.stateDiffDownloadTook(durationMillis); - - assert includeEphemeral == true; - // newState includes all the fields of cluster-state (includeEphemeral=true always) - remoteClusterStateCache.putState(clusterState); - return clusterState; } void validateClusterStateFromChecksum( @@ -2036,6 +2054,14 @@ public void diffDownloadFailed() { remoteStateStats.stateDiffDownloadFailed(); } + public void fullIncomingPublicationFailed() { + remoteStateStats.stateFullIncomingPublicationFailed(); + } + + public void diffIncomingPublicationFailed() { + remoteStateStats.stateDiffIncomingPublicationFailed(); + } + RemoteClusterStateCache getRemoteClusterStateCache() { return remoteClusterStateCache; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java index a8f4b33a19c37..0f520babca48d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDownloadStats.java @@ -20,10 +20,13 @@ public class RemoteDownloadStats extends PersistedStateStats { static final String CHECKSUM_VALIDATION_FAILED_COUNT = "checksum_validation_failed_count"; private AtomicLong checksumValidationFailedCount = new AtomicLong(0); + public static final String INCOMING_PUBLICATION_FAILED_COUNT = "incoming_publication_failed_count"; + private AtomicLong incomingPublicationFailedCount = new AtomicLong(0); public RemoteDownloadStats(String statsName) { super(statsName); addToExtendedFields(CHECKSUM_VALIDATION_FAILED_COUNT, checksumValidationFailedCount); + addToExtendedFields(INCOMING_PUBLICATION_FAILED_COUNT, incomingPublicationFailedCount); } public void checksumValidationFailedCount() { @@ -33,4 +36,12 @@ public void checksumValidationFailedCount() { public long getChecksumValidationFailedCount() { return checksumValidationFailedCount.get(); } + + public void incomingPublicationFailedCount() { + incomingPublicationFailedCount.incrementAndGet(); + } + + public long getIncomingPublicationFailedCount() { + return incomingPublicationFailedCount.get(); + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java index 1a8e85f30527d..7a6f5f9b95224 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistenceStats.java @@ -106,6 +106,14 @@ public long getStateFullDownloadValidationFailed() { return remoteFullDownloadStats.getChecksumValidationFailedCount(); } + public void stateDiffIncomingPublicationFailed() { + remoteDiffDownloadStats.incomingPublicationFailedCount(); + } + + public void stateFullIncomingPublicationFailed() { + remoteFullDownloadStats.incomingPublicationFailedCount(); + } + public PersistedStateStats getUploadStats() { return remoteUploadStats; } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 616559e91536d..c51f85e30283a 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -52,6 +52,7 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterStateDiffManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.RemoteDownloadStats; import org.opensearch.node.Node; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -64,10 +65,12 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import org.mockito.Mockito; +import static org.opensearch.gateway.remote.RemoteDownloadStats.INCOMING_PUBLICATION_FAILED_COUNT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.containsString; @@ -180,8 +183,8 @@ public void testHandleIncomingRemotePublishRequestWhenNoCurrentPublishRequest() () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - verify(remoteClusterStateService, times(0)).fullDownloadFailed(); - verify(remoteClusterStateService, times(1)).diffDownloadFailed(); + verify(remoteClusterStateService, times(0)).fullIncomingPublicationFailed(); + verify(remoteClusterStateService, times(1)).diffIncomingPublicationFailed(); verifyNoMoreInteractions(remoteClusterStateService); } @@ -207,8 +210,8 @@ public void testHandleIncomingRemotePublishRequestWhenTermMismatch() { () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - verify(remoteClusterStateService, times(0)).fullDownloadFailed(); - verify(remoteClusterStateService, times(1)).diffDownloadFailed(); + verify(remoteClusterStateService, times(0)).fullIncomingPublicationFailed(); + verify(remoteClusterStateService, times(1)).diffIncomingPublicationFailed(); verifyNoMoreInteractions(remoteClusterStateService); } @@ -234,8 +237,8 @@ public void testHandleIncomingRemotePublishRequestWhenVersionMismatch() { () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest) ); assertThat(e.getMessage(), containsString("publication to self failed")); - verify(remoteClusterStateService, times(1)).diffDownloadFailed(); - verify(remoteClusterStateService, times(0)).fullDownloadFailed(); + verify(remoteClusterStateService, times(1)).diffIncomingPublicationFailed(); + verify(remoteClusterStateService, times(0)).fullIncomingPublicationFailed(); verifyNoMoreInteractions(remoteClusterStateService); } @@ -263,20 +266,20 @@ public void testHandleIncomingRemotePublishRequestForLocalNode() throws IOExcept public void testDownloadRemotePersistedFullStateFailedStats() throws IOException { RemoteClusterStateService remoteClusterStateService = mock(RemoteClusterStateService.class); - PersistedStateStats remoteFullDownloadStats = new PersistedStateStats("dummy_full_stats"); - PersistedStateStats remoteDiffDownloadStats = new PersistedStateStats("dummy_diff_stats"); + PersistedStateStats remoteFullDownloadStats = new RemoteDownloadStats("dummy_full_stats"); + PersistedStateStats remoteDiffDownloadStats = new RemoteDownloadStats("dummy_diff_stats"); when(remoteClusterStateService.getFullDownloadStats()).thenReturn(remoteFullDownloadStats); when(remoteClusterStateService.getDiffDownloadStats()).thenReturn(remoteDiffDownloadStats); doAnswer((i) -> { - remoteFullDownloadStats.stateFailed(); + remoteFullDownloadStats.getExtendedFields().put(INCOMING_PUBLICATION_FAILED_COUNT, new AtomicLong(1)); return null; - }).when(remoteClusterStateService).fullDownloadFailed(); + }).when(remoteClusterStateService).fullIncomingPublicationFailed(); doAnswer((i) -> { - remoteDiffDownloadStats.stateFailed(); + remoteDiffDownloadStats.getExtendedFields().put(INCOMING_PUBLICATION_FAILED_COUNT, new AtomicLong(1)); return null; - }).when(remoteClusterStateService).diffDownloadFailed(); + }).when(remoteClusterStateService).diffIncomingPublicationFailed(); PublishWithJoinResponse expectedPublishResponse = new PublishWithJoinResponse(new PublishResponse(TERM, VERSION), Optional.empty()); Function handlePublishRequest = p -> expectedPublishResponse; @@ -294,8 +297,8 @@ public void testDownloadRemotePersistedFullStateFailedStats() throws IOException handler.setCurrentPublishRequestToSelf(publishRequest); assertThrows(IllegalStateException.class, () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest)); - assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); - assertEquals(0, remoteClusterStateService.getFullDownloadStats().getFailedCount()); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getExtendedFields().get(INCOMING_PUBLICATION_FAILED_COUNT).get()); + assertEquals(0, remoteClusterStateService.getFullDownloadStats().getExtendedFields().get(INCOMING_PUBLICATION_FAILED_COUNT).get()); } public void testDownloadRemotePersistedDiffStateFailedStats() throws IOException { @@ -309,9 +312,9 @@ public void testDownloadRemotePersistedDiffStateFailedStats() throws IOException when(remoteClusterStateService.getClusterMetadataManifestByFileName(any(), any())).thenReturn(metadataManifest); doAnswer((i) -> { - remoteDiffDownloadStats.stateFailed(); + remoteDiffDownloadStats.getExtendedFields().put(INCOMING_PUBLICATION_FAILED_COUNT, new AtomicLong(1)); return null; - }).when(remoteClusterStateService).diffDownloadFailed(); + }).when(remoteClusterStateService).diffIncomingPublicationFailed(); PublishWithJoinResponse expectedPublishResponse = new PublishWithJoinResponse(new PublishResponse(TERM, VERSION), Optional.empty()); Function handlePublishRequest = p -> expectedPublishResponse; @@ -333,7 +336,7 @@ public void testDownloadRemotePersistedDiffStateFailedStats() throws IOException handler.setCurrentPublishRequestToSelf(publishRequest); assertThrows(NullPointerException.class, () -> handler.handleIncomingRemotePublishRequest(remotePublishRequest)); - assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getExtendedFields().get(INCOMING_PUBLICATION_FAILED_COUNT).get()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 448b9cc9d78ac..be07aa0d05e9f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -962,6 +962,9 @@ public void testGetClusterStateForManifest_ExcludeEphemeral() throws IOException eq(false) ); + assertNotNull(remoteClusterStateService.getFullDownloadStats()); + assertEquals(1, remoteClusterStateService.getFullDownloadStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getFullDownloadStats().getFailedCount()); } public void testGetClusterStateFromManifest_CodecV1() throws IOException { @@ -1296,6 +1299,9 @@ public void testGetClusterStateUsingDiff() throws IOException { diffManifest.getClusterStateCustomDeleted().forEach(clusterStateCustomName -> { assertFalse(updatedClusterState.customs().containsKey(clusterStateCustomName)); }); + assertNotNull(remoteClusterStateService.getDiffDownloadStats()); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getSuccessCount()); + assertEquals(0, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); } public void testReadClusterStateInParallel_TimedOut() throws IOException { @@ -3421,6 +3427,9 @@ public void testGetClusterStateForManifestWithChecksumValidationEnabledWithMisma true ); assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateFullDownloadValidationFailed()); + assertNotNull(remoteClusterStateService.getFullDownloadStats()); + assertEquals(0, remoteClusterStateService.getFullDownloadStats().getSuccessCount()); + assertEquals(1, remoteClusterStateService.getFullDownloadStats().getFailedCount()); } public void testGetClusterStateForManifestWithChecksumValidationDebugWithMismatch() throws IOException { @@ -3717,6 +3726,9 @@ public void testGetClusterStateUsingDiffWithChecksumMismatch() throws IOExceptio eq(false) ); assertEquals(1, remoteClusterStateService.getRemoteStateStats().getStateDiffDownloadValidationFailed()); + assertNotNull(remoteClusterStateService.getDiffDownloadStats()); + assertEquals(0, remoteClusterStateService.getDiffDownloadStats().getSuccessCount()); + assertEquals(1, remoteClusterStateService.getDiffDownloadStats().getFailedCount()); } private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { From 9388217b4256c5c807b39dab47e536384643301e Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 22 Nov 2024 10:12:14 -0500 Subject: [PATCH 345/357] Update Gradle to 8.11.1 (#16694) Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3bfe9cc6bd3c2..ec480eaeb61ef 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=73d2d553933194d8eefed0a291acbe45392ca3572ba13834cbbf373da375276d +distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a From c82cd2ec76bf6a727bec8681b6f1b869f415fd31 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 22 Nov 2024 10:59:24 -0800 Subject: [PATCH 346/357] [Bugfix] Fix TieredSpilloverCache stats not adding correctly when shards are closed (#16560) * added draft tests for tsc stats holder Signed-off-by: Peter Alfonsi * first draft tsc stats bugfix Signed-off-by: Peter Alfonsi * Complete tests Signed-off-by: Peter Alfonsi * Cleanup Signed-off-by: Peter Alfonsi * Integrate fix with TSC Signed-off-by: Peter Alfonsi * Add IT Signed-off-by: Peter Alfonsi * Refactor cache package names in TSC module to match with server Signed-off-by: Peter Alfonsi * changelog Signed-off-by: Peter Alfonsi * Revert "Refactor cache package names in TSC module to match with server" This reverts commit 3b15a7a4795b7638deb2998cd3d060d5a87e26a1. Signed-off-by: Peter Alfonsi * Addressed Sagar's comments Signed-off-by: Peter Alfonsi * More package fixes Signed-off-by: Peter Alfonsi * Addressed andross's comments Signed-off-by: Peter Alfonsi --------- Signed-off-by: Peter Alfonsi Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi --- CHANGELOG.md | 1 + .../tier/TieredSpilloverCacheStatsIT.java | 51 +++ .../common/tier/TieredSpilloverCache.java | 10 +- .../tier/TieredSpilloverCacheStatsHolder.java | 15 + .../TieredSpilloverCacheStatsHolderTests.java | 378 ++++++++++++++++++ .../tier/TieredSpilloverCacheTests.java | 54 +++ .../cache/stats/DefaultCacheStatsHolder.java | 17 +- 7 files changed, 512 insertions(+), 14 deletions(-) create mode 100644 modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolderTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cfcd4e6dfbd1..70245afda0dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544)) - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628)) - Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670)) +- [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560)) ### Security diff --git a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java index fe6bd7050a8f3..a858e94ad1609 100644 --- a/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java +++ b/modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsIT.java @@ -10,6 +10,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.search.SearchResponse; @@ -40,6 +41,7 @@ import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP; +import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; @@ -417,6 +419,55 @@ public void testStatsWithMultipleSegments() throws Exception { assertTrue(diskCacheStat.getEvictions() == 0); } + public void testClosingShard() throws Exception { + // Closing the shard should totally remove the stats associated with that shard. + internalCluster().startNodes( + 1, + Settings.builder() + .put(defaultSettings(HEAP_CACHE_SIZE_STRING, getNumberOfSegments())) + .put( + TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), + new TimeValue(0, TimeUnit.SECONDS) + ) + .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1)) + .build() + ); + String index = "index"; + Client client = client(); + startIndex(client, index); + + // First search one time to see how big a single value will be + searchIndex(client, index, 0); + // get total stats + long singleSearchSize = getTotalStats(client).getSizeInBytes(); + // Select numbers so we get some values on both heap and disk + int itemsOnHeap = HEAP_CACHE_SIZE / (int) singleSearchSize; + int itemsOnDisk = 1 + randomInt(30); // The first one we search (to get the size) always goes to disk + int expectedEntries = itemsOnHeap + itemsOnDisk; + + for (int i = 1; i < expectedEntries; i++) { + // Cause misses + searchIndex(client, index, i); + } + int expectedMisses = itemsOnHeap + itemsOnDisk; + + // Cause some hits + int expectedHits = randomIntBetween(itemsOnHeap, expectedEntries); // Select it so some hits come from both tiers + for (int i = 0; i < expectedHits; i++) { + searchIndex(client, index, i); + } + + // Check the new stats API values are as expected + assertEquals( + new ImmutableCacheStats(expectedHits, expectedMisses, 0, expectedEntries * singleSearchSize, expectedEntries), + getTotalStats(client) + ); + + // Closing the index should close the shard + assertAcked(client().admin().indices().delete(new DeleteIndexRequest("index")).get()); + assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), getTotalStats(client)); + } + private void startIndex(Client client, String indexName) throws InterruptedException { assertAcked( client.admin() diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java index ab5335ca0ca66..38a6915ffd10e 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java @@ -373,12 +373,10 @@ private V compute(ICacheKey key, LoadAwareCacheLoader, V> loader @Override public void invalidate(ICacheKey key) { - for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { - if (key.getDropStatsForDimensions()) { - List dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, cacheEntry.getValue().tierName); - statsHolder.removeDimensions(dimensionValues); - } - if (key.key != null) { + if (key.getDropStatsForDimensions()) { + statsHolder.removeDimensions(key.dimensions); + } else if (key.key != null) { + for (Map.Entry, TierInfo> cacheEntry : caches.entrySet()) { try (ReleasableLock ignore = writeLock.acquire()) { cacheEntry.getKey().invalidate(key); } diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java index b40724430454b..7ea6d3504a52c 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java @@ -43,6 +43,8 @@ public class TieredSpilloverCacheStatsHolder extends DefaultCacheStatsHolder { /** Dimension value for on-disk cache, like EhcacheDiskCache. */ public static final String TIER_DIMENSION_VALUE_DISK = "disk"; + static final List TIER_VALUES = List.of(TIER_DIMENSION_VALUE_ON_HEAP, TIER_DIMENSION_VALUE_DISK); + /** * Constructor for the stats holder. * @param originalDimensionNames the original dimension names, not including TIER_DIMENSION_NAME @@ -167,4 +169,17 @@ public void decrementItems(List dimensionValues) { void setDiskCacheEnabled(boolean diskCacheEnabled) { this.diskCacheEnabled = diskCacheEnabled; } + + @Override + public void removeDimensions(List dimensionValues) { + assert dimensionValues.size() == dimensionNames.size() - 1 + : "Must specify a value for every dimension except tier when removing from StatsHolder"; + // As we are removing nodes from the tree, obtain the lock + lock.lock(); + try { + removeDimensionsHelper(dimensionValues, statsRoot, 0); + } finally { + lock.unlock(); + } + } } diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolderTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolderTests.java new file mode 100644 index 0000000000000..09273a0761663 --- /dev/null +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolderTests.java @@ -0,0 +1,378 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache.common.tier; + +import org.opensearch.common.Randomness; +import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.common.cache.stats.DefaultCacheStatsHolder; +import org.opensearch.common.cache.stats.ImmutableCacheStats; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; + +import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP; +import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_VALUES; + +public class TieredSpilloverCacheStatsHolderTests extends OpenSearchTestCase { + // These are modified from DefaultCacheStatsHolderTests.java to account for the tiers. Because we can't add a dependency on server.test, + // we can't reuse the same code. + + public void testAddAndGet() throws Exception { + for (boolean diskTierEnabled : List.of(true, false)) { + List dimensionNames = List.of("dim1", "dim2", "dim3", "dim4"); + TieredSpilloverCacheStatsHolder cacheStatsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, diskTierEnabled); + Map> usedDimensionValues = getUsedDimensionValues(cacheStatsHolder, 10, diskTierEnabled); + Map, CacheStats> expected = populateStats(cacheStatsHolder, usedDimensionValues, 1000, 10, diskTierEnabled); + + // test the value in the map is as expected for each distinct combination of values (all leaf nodes) + for (List dimensionValues : expected.keySet()) { + CacheStats expectedCounter = expected.get(dimensionValues); + ImmutableCacheStats actualStatsHolder = getNode(dimensionValues, cacheStatsHolder.getStatsRoot()).getImmutableStats(); + ImmutableCacheStats actualCacheStats = getNode(dimensionValues, cacheStatsHolder.getStatsRoot()).getImmutableStats(); + assertEquals(expectedCounter.immutableSnapshot(), actualStatsHolder); + assertEquals(expectedCounter.immutableSnapshot(), actualCacheStats); + } + + // Check overall total matches + CacheStats expectedTotal = new CacheStats(); + for (List dims : expected.keySet()) { + CacheStats other = expected.get(dims); + boolean countMissesAndEvictionsTowardsTotal = dims.get(dims.size() - 1).equals(TIER_DIMENSION_VALUE_DISK) + || !diskTierEnabled; + add(expectedTotal, other, countMissesAndEvictionsTowardsTotal); + } + assertEquals(expectedTotal.immutableSnapshot(), cacheStatsHolder.getStatsRoot().getImmutableStats()); + } + } + + private void add(CacheStats original, CacheStats other, boolean countMissesAndEvictionsTowardsTotal) { + // Add other to original, accounting for whether other is from the heap or disk tier + long misses = 0; + long evictions = 0; + if (countMissesAndEvictionsTowardsTotal) { + misses = other.getMisses(); + evictions = other.getEvictions(); + } + CacheStats modifiedOther = new CacheStats(other.getHits(), misses, evictions, other.getSizeInBytes(), other.getItems()); + original.add(modifiedOther); + } + + public void testReset() throws Exception { + List dimensionNames = List.of("dim1", "dim2"); + TieredSpilloverCacheStatsHolder cacheStatsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, true); + Map> usedDimensionValues = getUsedDimensionValues(cacheStatsHolder, 10, true); + Map, CacheStats> expected = populateStats(cacheStatsHolder, usedDimensionValues, 100, 10, true); + + cacheStatsHolder.reset(); + for (List dimensionValues : expected.keySet()) { + CacheStats originalCounter = expected.get(dimensionValues); + ImmutableCacheStats expectedTotal = new ImmutableCacheStats( + originalCounter.getHits(), + originalCounter.getMisses(), + originalCounter.getEvictions(), + 0, + 0 + ); + + DefaultCacheStatsHolder.Node node = getNode(dimensionValues, cacheStatsHolder.getStatsRoot()); + ImmutableCacheStats actual = node.getImmutableStats(); + assertEquals(expectedTotal, actual); + } + } + + public void testDropStatsForDimensions() throws Exception { + List dimensionNames = List.of("dim1", "dim2"); + // Create stats for the following dimension sets + List> statsToPopulate = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3")); + for (boolean diskTierEnabled : List.of(true, false)) { + TieredSpilloverCacheStatsHolder cacheStatsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, diskTierEnabled); + setupRemovalTest(cacheStatsHolder, statsToPopulate, diskTierEnabled); + + // Check the resulting total is correct. + int numNodes = statsToPopulate.size(); // Number of distinct sets of dimensions (not including tiers) + // If disk tier is enabled, we expect hits to be 2 * numNodes (1 heap + 1 disk per combination of dims), otherwise 1 * numNodes. + // Misses and evictions should be 1 * numNodes in either case (if disk tier is present, count only the disk misses/evictions, if + // disk tier is absent, count the heap ones) + long originalHits = diskTierEnabled ? 2 * numNodes : numNodes; + ImmutableCacheStats expectedTotal = new ImmutableCacheStats(originalHits, numNodes, numNodes, 0, 0); + assertEquals(expectedTotal, cacheStatsHolder.getStatsRoot().getImmutableStats()); + + // When we invalidate A2, B2, we should lose the node for B2, but not B3 or A2. + cacheStatsHolder.removeDimensions(List.of("A2", "B2")); + + // We expect hits to go down by 2 (1 heap + 1 disk) if disk is enabled, and 1 otherwise. Evictions/misses should go down by 1 in + // either case. + long removedHitsPerRemovedNode = diskTierEnabled ? 2 : 1; + expectedTotal = new ImmutableCacheStats(originalHits - removedHitsPerRemovedNode, numNodes - 1, numNodes - 1, 0, 0); + assertEquals(expectedTotal, cacheStatsHolder.getStatsRoot().getImmutableStats()); + assertNull(getNode(List.of("A2", "B2", TIER_DIMENSION_VALUE_ON_HEAP), cacheStatsHolder.getStatsRoot())); + assertNull(getNode(List.of("A2", "B2", TIER_DIMENSION_VALUE_DISK), cacheStatsHolder.getStatsRoot())); + assertNull(getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot())); + assertNotNull(getNode(List.of("A2"), cacheStatsHolder.getStatsRoot())); + assertNotNull(getNode(List.of("A2", "B3", TIER_DIMENSION_VALUE_ON_HEAP), cacheStatsHolder.getStatsRoot())); + + // When we invalidate A1, B1, we should lose the nodes for B1 and also A1, as it has no more children. + cacheStatsHolder.removeDimensions(List.of("A1", "B1")); + expectedTotal = new ImmutableCacheStats(originalHits - 2 * removedHitsPerRemovedNode, numNodes - 2, numNodes - 2, 0, 0); + assertEquals(expectedTotal, cacheStatsHolder.getStatsRoot().getImmutableStats()); + assertNull(getNode(List.of("A1", "B1", TIER_DIMENSION_VALUE_ON_HEAP), cacheStatsHolder.getStatsRoot())); + assertNull(getNode(List.of("A1", "B1", TIER_DIMENSION_VALUE_DISK), cacheStatsHolder.getStatsRoot())); + assertNull(getNode(List.of("A1", "B1"), cacheStatsHolder.getStatsRoot())); + assertNull(getNode(List.of("A1"), cacheStatsHolder.getStatsRoot())); + + // When we invalidate the last node, all nodes should be deleted except the root node + cacheStatsHolder.removeDimensions(List.of("A2", "B3")); + assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), cacheStatsHolder.getStatsRoot().getImmutableStats()); + // assertEquals(0, cacheStatsHolder.getStatsRoot().getChildren().size()); + } + } + + public void testCount() throws Exception { + List dimensionNames = List.of("dim1", "dim2"); + TieredSpilloverCacheStatsHolder cacheStatsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, true); + Map> usedDimensionValues = getUsedDimensionValues(cacheStatsHolder, 10, true); + Map, CacheStats> expected = populateStats(cacheStatsHolder, usedDimensionValues, 100, 10, true); + + long expectedCount = 0L; + for (CacheStats counter : expected.values()) { + expectedCount += counter.getItems(); + } + assertEquals(expectedCount, cacheStatsHolder.count()); + } + + public void testConcurrentRemoval() throws Exception { + List dimensionNames = List.of("A", "B"); + TieredSpilloverCacheStatsHolder cacheStatsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, true); + + // Create stats for the following dimension sets + List> statsToPopulate = new ArrayList<>(); + int numAValues = 10; + int numBValues = 2; + for (int indexA = 0; indexA < numAValues; indexA++) { + for (int indexB = 0; indexB < numBValues; indexB++) { + statsToPopulate.add(List.of("A" + indexA, "B" + indexB)); + } + } + setupRemovalTest(cacheStatsHolder, statsToPopulate, true); + + // Remove a subset of the dimensions concurrently. + // Remove both (A0, B0), and (A0, B1), so we expect the intermediate node for A0 to be null afterwards. + // For all the others, remove only the B0 value. Then we expect the intermediate nodes for A1 through A9 to be present + // and reflect only the stats for their B1 child. + + Thread[] threads = new Thread[numAValues + 1]; + for (int i = 0; i < numAValues; i++) { + int finalI = i; + threads[i] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A" + finalI, "B0")); }); + } + threads[numAValues] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A0", "B1")); }); + for (Thread thread : threads) { + thread.start(); + } + for (Thread thread : threads) { + thread.join(); + } + + // intermediate node for A0 should be null + assertNull(getNode(List.of("A0"), cacheStatsHolder.getStatsRoot())); + + // leaf nodes for all B0 values should be null since they were removed + for (int indexA = 0; indexA < numAValues; indexA++) { + assertNull(getNode(List.of("A" + indexA, "B0"), cacheStatsHolder.getStatsRoot())); + } + + // leaf nodes for all B1 values, except (A0, B1), should not be null as they weren't removed, + // and the intermediate nodes A1 through A9 shouldn't be null as they have remaining children + for (int indexA = 1; indexA < numAValues; indexA++) { + DefaultCacheStatsHolder.Node b1LeafNode = getNode(List.of("A" + indexA, "B1"), cacheStatsHolder.getStatsRoot()); + assertNotNull(b1LeafNode); + assertEquals(new ImmutableCacheStats(2, 1, 1, 0, 0), b1LeafNode.getImmutableStats()); + DefaultCacheStatsHolder.Node intermediateLevelNode = getNode(List.of("A" + indexA), cacheStatsHolder.getStatsRoot()); + assertNotNull(intermediateLevelNode); + assertEquals(b1LeafNode.getImmutableStats(), intermediateLevelNode.getImmutableStats()); + } + } + + static void setupRemovalTest( + TieredSpilloverCacheStatsHolder cacheStatsHolder, + List> statsToPopulate, + boolean diskTierEnabled + ) { + List tiers = diskTierEnabled ? TIER_VALUES : List.of(TIER_DIMENSION_VALUE_ON_HEAP); + for (List dims : statsToPopulate) { + // Increment hits, misses, and evictions for set of dimensions, for both heap and disk + for (String tier : tiers) { + List dimsWithDimension = cacheStatsHolder.getDimensionsWithTierValue(dims, tier); + cacheStatsHolder.incrementHits(dimsWithDimension); + cacheStatsHolder.incrementMisses(dimsWithDimension); + boolean includeInTotal = tier.equals(TIER_DIMENSION_VALUE_DISK) || !diskTierEnabled; + cacheStatsHolder.incrementEvictions(dimsWithDimension, includeInTotal); + } + } + } + + /** + * Returns the node found by following these dimension values down from the root node. + * Returns null if no such node exists. + */ + static DefaultCacheStatsHolder.Node getNode(List dimensionValues, DefaultCacheStatsHolder.Node root) { + DefaultCacheStatsHolder.Node current = root; + for (String dimensionValue : dimensionValues) { + current = current.getChildren().get(dimensionValue); + if (current == null) { + return null; + } + } + return current; + } + + static Map, CacheStats> populateStats( + TieredSpilloverCacheStatsHolder cacheStatsHolder, + Map> usedDimensionValues, + int numDistinctValuePairs, + int numRepetitionsPerValue, + boolean diskTierEnabled + ) throws InterruptedException { + return populateStats( + List.of(cacheStatsHolder), + usedDimensionValues, + numDistinctValuePairs, + numRepetitionsPerValue, + diskTierEnabled + ); + } + + static Map, CacheStats> populateStats( + List cacheStatsHolders, + Map> usedDimensionValues, + int numDistinctValuePairs, + int numRepetitionsPerValue, + boolean diskTierEnabled + ) throws InterruptedException { + for (TieredSpilloverCacheStatsHolder statsHolder : cacheStatsHolders) { + assertEquals(cacheStatsHolders.get(0).getDimensionNames(), statsHolder.getDimensionNames()); + } + Map, CacheStats> expected = new ConcurrentHashMap<>(); + Thread[] threads = new Thread[numDistinctValuePairs]; + CountDownLatch countDownLatch = new CountDownLatch(numDistinctValuePairs); + Random rand = Randomness.get(); + List> dimensionsForThreads = new ArrayList<>(); + for (int i = 0; i < numDistinctValuePairs; i++) { + dimensionsForThreads.add(getRandomDimList(cacheStatsHolders.get(0).getDimensionNames(), usedDimensionValues, true, rand)); + int finalI = i; + threads[i] = new Thread(() -> { + Random threadRand = Randomness.get(); + List dimensions = dimensionsForThreads.get(finalI); + expected.computeIfAbsent(dimensions, (key) -> new CacheStats()); + for (TieredSpilloverCacheStatsHolder cacheStatsHolder : cacheStatsHolders) { + for (int j = 0; j < numRepetitionsPerValue; j++) { + CacheStats statsToInc = new CacheStats( + threadRand.nextInt(10), + threadRand.nextInt(10), + threadRand.nextInt(10), + threadRand.nextInt(5000), + threadRand.nextInt(10) + ); + for (int iter = 0; iter < statsToInc.getHits(); iter++) { + expected.get(dimensions).incrementHits(); + } + for (int iter = 0; iter < statsToInc.getMisses(); iter++) { + expected.get(dimensions).incrementMisses(); + } + for (int iter = 0; iter < statsToInc.getEvictions(); iter++) { + expected.get(dimensions).incrementEvictions(); + } + expected.get(dimensions).incrementSizeInBytes(statsToInc.getSizeInBytes()); + for (int iter = 0; iter < statsToInc.getItems(); iter++) { + expected.get(dimensions).incrementItems(); + } + populateStatsHolderFromStatsValueMap(cacheStatsHolder, Map.of(dimensions, statsToInc), diskTierEnabled); + } + } + countDownLatch.countDown(); + }); + } + for (Thread thread : threads) { + thread.start(); + } + countDownLatch.await(); + return expected; + } + + private static List getRandomDimList( + List dimensionNames, + Map> usedDimensionValues, + boolean pickValueForAllDims, + Random rand + ) { + List result = new ArrayList<>(); + for (String dimName : dimensionNames) { + if (pickValueForAllDims || rand.nextBoolean()) { // if pickValueForAllDims, always pick a value for each dimension, otherwise do + // so 50% of the time + int index = between(0, usedDimensionValues.get(dimName).size() - 1); + result.add(usedDimensionValues.get(dimName).get(index)); + } + } + return result; + } + + static Map> getUsedDimensionValues( + TieredSpilloverCacheStatsHolder cacheStatsHolder, + int numValuesPerDim, + boolean diskTierEnabled + ) { + Map> usedDimensionValues = new HashMap<>(); + for (int i = 0; i < cacheStatsHolder.getDimensionNames().size() - 1; i++) { // Have to handle final tier dimension separately + List values = new ArrayList<>(); + for (int j = 0; j < numValuesPerDim; j++) { + values.add(UUID.randomUUID().toString()); + } + usedDimensionValues.put(cacheStatsHolder.getDimensionNames().get(i), values); + } + if (diskTierEnabled) { + usedDimensionValues.put(TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME, TIER_VALUES); + } else { + usedDimensionValues.put(TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME, List.of(TIER_DIMENSION_VALUE_ON_HEAP)); + } + return usedDimensionValues; + } + + public static void populateStatsHolderFromStatsValueMap( + TieredSpilloverCacheStatsHolder cacheStatsHolder, + Map, CacheStats> statsMap, + boolean diskTierEnabled + ) { + for (Map.Entry, CacheStats> entry : statsMap.entrySet()) { + CacheStats stats = entry.getValue(); + List dims = entry.getKey(); + for (int i = 0; i < stats.getHits(); i++) { + cacheStatsHolder.incrementHits(dims); + } + for (int i = 0; i < stats.getMisses(); i++) { + cacheStatsHolder.incrementMisses(dims); + } + for (int i = 0; i < stats.getEvictions(); i++) { + boolean includeInTotal = dims.get(dims.size() - 1).equals(TIER_DIMENSION_VALUE_DISK) || !diskTierEnabled; + cacheStatsHolder.incrementEvictions(dims, includeInTotal); + } + cacheStatsHolder.incrementSizeInBytes(dims, stats.getSizeInBytes()); + for (int i = 0; i < stats.getItems(); i++) { + cacheStatsHolder.incrementItems(dims); + } + } + } +} diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java index 1215a2130ac2d..3bb1321f9faf2 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java @@ -2112,6 +2112,60 @@ public void testTieredCacheDefaultSegmentCount() { assertTrue(VALID_SEGMENT_COUNT_VALUES.contains(tieredSpilloverCache.getNumberOfSegments())); } + public void testDropStatsForDimensions() throws Exception { + int onHeapCacheSize = randomIntBetween(300, 600); + int diskCacheSize = randomIntBetween(700, 1200); + int numberOfSegments = getNumberOfSegments(); + int keyValueSize = 50; + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + Settings.builder() + .put( + TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace( + CacheType.INDICES_REQUEST_CACHE.getSettingPrefix() + ).getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(), + 0, + numberOfSegments + ); + + List> usedKeys = new ArrayList<>(); + // Fill the cache, getting some entries + evictions for both tiers + int minMisses = (diskCacheSize + onHeapCacheSize) / keyValueSize + 10; + int numMisses = onHeapCacheSize + diskCacheSize + randomIntBetween(minMisses, minMisses + 50); + for (int iter = 0; iter < numMisses; iter++) { + ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + usedKeys.add(key); + LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); + tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader); + } + // Also do some random hits + Random rand = Randomness.get(); + int approxNumHits = 30; + for (int i = 0; i < approxNumHits; i++) { + LoadAwareCacheLoader, String> tieredCacheLoader = getLoadAwareCacheLoader(); + ICacheKey key = usedKeys.get(rand.nextInt(usedKeys.size())); + tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader); + } + + ImmutableCacheStats totalStats = tieredSpilloverCache.stats().getTotalStats(); + assertTrue(totalStats.getHits() > 0); + assertTrue(totalStats.getMisses() > 0); + assertTrue(totalStats.getEvictions() > 0); + + // Since all the keys have the same dimension values, except tiers, we only need to remove that one, and we expect all stats values + // should be 0 after that. + ICacheKey dropDimensionsKey = new ICacheKey<>(null, getMockDimensions()); + dropDimensionsKey.setDropStatsForDimensions(true); + tieredSpilloverCache.invalidate(dropDimensionsKey); + assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.stats().getTotalStats()); + } + private List getMockDimensions() { List dims = new ArrayList<>(); for (String dimensionName : dimensionNames) { diff --git a/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java index ea92c8e81b8f0..7434283ff6f41 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java @@ -37,10 +37,10 @@ public class DefaultCacheStatsHolder implements CacheStatsHolder { // Non-leaf nodes have stats matching the sum of their children. // We use a tree structure, rather than a map with concatenated keys, to save on memory usage. If there are many leaf // nodes that share a parent, that parent's dimension value will only be stored once, not many times. - private final Node statsRoot; + protected final Node statsRoot; // To avoid sync problems, obtain a lock before creating or removing nodes in the stats tree. // No lock is needed to edit stats on existing nodes. - private final Lock lock = new ReentrantLock(); + protected final Lock lock = new ReentrantLock(); // The name of the cache type using these stats private final String storeName; @@ -188,8 +188,10 @@ public void removeDimensions(List dimensionValues) { } // Returns a CacheStatsCounterSnapshot object for the stats to decrement if the removal happened, null otherwise. - private ImmutableCacheStats removeDimensionsHelper(List dimensionValues, Node node, int depth) { + protected ImmutableCacheStats removeDimensionsHelper(List dimensionValues, Node node, int depth) { if (depth == dimensionValues.size()) { + // Remove children, if present. + node.children.clear(); // Pass up a snapshot of the original stats to avoid issues when the original is decremented by other fn invocations return node.getImmutableStats(); } @@ -208,15 +210,14 @@ private ImmutableCacheStats removeDimensionsHelper(List dimensionValues, return statsToDecrement; } - // pkg-private for testing - Node getStatsRoot() { + public Node getStatsRoot() { return statsRoot; } /** * Nodes that make up the tree in the stats holder. */ - protected static class Node { + public static class Node { private final String dimensionValue; // Map from dimensionValue to the DimensionNode for that dimension value. final Map children; @@ -241,7 +242,7 @@ public String getDimensionValue() { return dimensionValue; } - protected Map getChildren() { + public Map getChildren() { // We can safely iterate over ConcurrentHashMap without worrying about thread issues. return children; } @@ -280,7 +281,7 @@ long getEntries() { return this.stats.getItems(); } - ImmutableCacheStats getImmutableStats() { + public ImmutableCacheStats getImmutableStats() { return this.stats.immutableSnapshot(); } From 3da97f24ed0fbf90d403ba70d8a09d3850acb94d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:16:06 -0500 Subject: [PATCH 347/357] Bump org.apache.logging.log4j:log4j-core from 2.24.1 to 2.24.2 in /buildSrc/src/testKit/thirdPartyAudit (#16718) * Bump org.apache.logging.log4j:log4j-core Bumps org.apache.logging.log4j:log4j-core from 2.24.1 to 2.24.2. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70245afda0dd1..e544b860d027a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `Netty` from 4.1.114.Final to 4.1.115.Final ([#16661](https://github.com/opensearch-project/OpenSearch/pull/16661)) - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) - Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667)) +- Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718)) ### Changed diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle index 4d425964c77af..3db2a6e7c2733 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle @@ -17,7 +17,7 @@ repositories { } dependencies { - implementation "org.apache.logging.log4j:log4j-core:2.24.1" + implementation "org.apache.logging.log4j:log4j-core:2.24.2" } ["0.0.1", "0.0.2"].forEach { v -> From 581771053c324e0774c3309ec9806d53d99f9f35 Mon Sep 17 00:00:00 2001 From: Dmitry Kryukov Date: Wed, 27 Nov 2024 00:38:19 +0300 Subject: [PATCH 348/357] Removed unused array (#15364) Signed-off-by: Dmitry Kryukov --- .../search/aggregations/bucket/BucketsAggregator.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java index eef427754f535..5420d8c7f6dbf 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java @@ -221,10 +221,6 @@ protected final InternalAggregations[] buildSubAggsForBuckets(long[] bucketOrdsT } InternalAggregations[] result = new InternalAggregations[bucketOrdsToCollect.length]; for (int ord = 0; ord < bucketOrdsToCollect.length; ord++) { - InternalAggregation[] slice = new InternalAggregation[subAggregators.length]; - for (int i = 0; i < subAggregators.length; i++) { - slice[i] = aggregations[i][ord]; - } final int thisOrd = ord; result[ord] = InternalAggregations.from(new AbstractList() { @Override From ad982c20e089cbf6a2897c1f3e3bfc3b64f768da Mon Sep 17 00:00:00 2001 From: Mikhail Khludnev Date: Thu, 28 Nov 2024 01:50:10 +0300 Subject: [PATCH 349/357] Support more than 1024 IP/masks with indexed field (#16391) MultiRangeQuery for searching IP masks more 1025 masks in indexed field. --------- Signed-off-by: Mikhail Khludnev Signed-off-by: Mikhail Khludnev Co-authored-by: Andriy Redko --- CHANGELOG.md | 1 + .../search/fields/SearchIpFieldTermsIT.java | 243 ++++++++++++++++++ .../index/mapper/IpFieldMapper.java | 149 ++++++++--- .../index/mapper/IpFieldTypeTests.java | 22 +- 4 files changed, 375 insertions(+), 40 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/fields/SearchIpFieldTermsIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e544b860d027a..c81586548d210 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718)) ### Changed +- Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391) ### Deprecated diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchIpFieldTermsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchIpFieldTermsIT.java new file mode 100644 index 0000000000000..d28d07a07e785 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchIpFieldTermsIT.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.fields; + +import org.apache.lucene.search.IndexSearcher; +import org.opensearch.action.bulk.BulkRequestBuilder; +import org.opensearch.action.search.SearchPhaseExecutionException; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.test.OpenSearchSingleNodeTestCase; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; + +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.hamcrest.Matchers.equalTo; + +public class SearchIpFieldTermsIT extends OpenSearchSingleNodeTestCase { + + /** + * @return number of expected matches + * */ + private int createIndex(String indexName, int numberOfMasks, List queryTermsSink) throws IOException { + XContentBuilder xcb = createMapping(); + client().admin().indices().prepareCreate(indexName).setMapping(xcb).get(); + ensureGreen(); + + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + + Set dedupeCidrs = new HashSet<>(); + int cidrs = 0; + int ips = 0; + + for (int i = 0; ips <= 10240 && cidrs < numberOfMasks && i < 1000000; i++) { + String ip; + int prefix; + boolean mask; + do { + mask = ips > 0 && random().nextBoolean(); + ip = generateRandomIPv4(); + prefix = 24 + random().nextInt(8); // CIDR prefix for IPv4 + } while (mask && !dedupeCidrs.add(getFirstThreeOctets(ip))); + + bulkRequestBuilder.add( + client().prepareIndex(indexName).setSource(Map.of("addr", ip, "dummy_filter", randomSubsetOf(1, "1", "2", "3"))) + ); + + final String termToQuery; + if (mask) { + termToQuery = ip + "/" + prefix; + cidrs++; + } else { + termToQuery = ip; + ips++; + } + queryTermsSink.add(termToQuery); + } + int addMatches = 0; + for (int i = 0; i < atLeast(100); i++) { + final String ip; + ip = generateRandomIPv4(); + bulkRequestBuilder.add( + client().prepareIndex(indexName).setSource(Map.of("addr", ip, "dummy_filter", randomSubsetOf(1, "1", "2", "3"))) + ); + boolean match = false; + for (String termQ : queryTermsSink) { + boolean isCidr = termQ.contains("/"); + if ((isCidr && isIPInCIDR(ip, termQ)) || (!isCidr && termQ.equals(ip))) { + match = true; + break; + } + } + if (match) { + addMatches++; + } else { + break; // single mismatch is enough. + } + } + + bulkRequestBuilder.setRefreshPolicy(IMMEDIATE).get(); + return ips + cidrs + addMatches; + } + + public void testLessThanMaxClauses() throws IOException { + ArrayList toQuery = new ArrayList<>(); + String indexName = "small"; + int expectMatches = createIndex(indexName, IndexSearcher.getMaxClauseCount() - 1, toQuery); + + assertTermsHitCount(indexName, "addr", toQuery, expectMatches); + assertTermsHitCount(indexName, "addr.idx", toQuery, expectMatches); + assertTermsHitCount(indexName, "addr.dv", toQuery, expectMatches); + // passing dummy filter crushes on rewriting + SearchPhaseExecutionException ose = assertThrows(SearchPhaseExecutionException.class, () -> { + assertTermsHitCount( + indexName, + "addr.dv", + toQuery, + expectMatches, + (boolBuilder) -> boolBuilder.filter(QueryBuilders.termsQuery("dummy_filter", "1", "2", "3")) + .filter(QueryBuilders.termsQuery("dummy_filter", "1", "2", "3", "4")) + .filter(QueryBuilders.termsQuery("dummy_filter", "1", "2", "3", "4", "5")) + ); + }); + assertTrue("exceeding on query rewrite", ose.shardFailures()[0].getCause() instanceof IndexSearcher.TooManyNestedClauses); + } + + public void testExceedMaxClauses() throws IOException { + ArrayList toQuery = new ArrayList<>(); + String indexName = "larger"; + int expectMatches = createIndex(indexName, IndexSearcher.getMaxClauseCount() + (rarely() ? 0 : atLeast(10)), toQuery); + assertTermsHitCount(indexName, "addr", toQuery, expectMatches); + assertTermsHitCount(indexName, "addr.idx", toQuery, expectMatches); + // error from mapper/parser + final SearchPhaseExecutionException ose = assertThrows( + SearchPhaseExecutionException.class, + () -> assertTermsHitCount(indexName, "addr.dv", toQuery, expectMatches) + ); + assertTrue("exceeding on query building", ose.shardFailures()[0].getCause().getCause() instanceof IndexSearcher.TooManyClauses); + } + + private static String getFirstThreeOctets(String ipAddress) { + // Split the IP address by the dot delimiter + String[] octets = ipAddress.split("\\."); + + // Take the first three octets + String[] firstThreeOctets = new String[3]; + System.arraycopy(octets, 0, firstThreeOctets, 0, 3); + + // Join the first three octets back together with dots + return String.join(".", firstThreeOctets); + } + + private void assertTermsHitCount(String indexName, String field, Collection toQuery, long expectedMatches) { + assertTermsHitCount(indexName, field, toQuery, expectedMatches, (bqb) -> {}); + } + + private void assertTermsHitCount( + String indexName, + String field, + Collection toQuery, + long expectedMatches, + Consumer addFilter + ) { + TermsQueryBuilder ipTerms = QueryBuilders.termsQuery(field, new ArrayList<>(toQuery)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + addFilter.accept(boolQueryBuilder); + SearchResponse result = client().prepareSearch(indexName).setQuery(boolQueryBuilder.must(ipTerms) + // .filter(QueryBuilders.termsQuery("dummy_filter", "a", "b")) + ).get(); + long hitsFound = Objects.requireNonNull(result.getHits().getTotalHits()).value; + MatcherAssert.assertThat(field, hitsFound, equalTo(expectedMatches)); + } + + // Converts an IP string (either IPv4 or IPv6) to a byte array + private static byte[] ipToBytes(String ip) { + InetAddress inetAddress = InetAddresses.forString(ip); + return inetAddress.getAddress(); + } + + // Checks if an IP is within a given CIDR (works for both IPv4 and IPv6) + private static boolean isIPInCIDR(String ip, String cidr) { + String[] cidrParts = cidr.split("/"); + String cidrIp = cidrParts[0]; + int prefixLength = Integer.parseInt(cidrParts[1]); + + byte[] ipBytes = ipToBytes(ip); + byte[] cidrIpBytes = ipToBytes(cidrIp); + + // Calculate how many full bytes and how many bits are in the mask + int fullBytes = prefixLength / 8; + int extraBits = prefixLength % 8; + + // Compare full bytes + for (int i = 0; i < fullBytes; i++) { + if (ipBytes[i] != cidrIpBytes[i]) { + return false; + } + } + + // Compare extra bits (if any) + if (extraBits > 0) { + int mask = 0xFF << (8 - extraBits); + return (ipBytes[fullBytes] & mask) == (cidrIpBytes[fullBytes] & mask); + } + + return true; + } + + // Generate a random IPv4 address + private String generateRandomIPv4() { + return String.join( + ".", + String.valueOf(random().nextInt(256)), + String.valueOf(random().nextInt(256)), + String.valueOf(random().nextInt(256)), + String.valueOf(random().nextInt(256)) + ); + } + + private XContentBuilder createMapping() throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("addr") + .field("type", "ip") + .startObject("fields") + .startObject("idx") + .field("type", "ip") + .field("doc_values", false) + .endObject() + .startObject("dv") + .field("type", "ip") + .field("index", false) + .endObject() + .endObject() + .endObject() + .startObject("dummy_filter") + .field("type", "keyword") + .endObject() + .endObject() + .endObject(); + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index c51cada9f3143..e23a48f94f450 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -36,6 +36,10 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.sandbox.search.MultiRangeQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; @@ -47,6 +51,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.network.InetAddresses; +import org.opensearch.common.network.NetworkAddress; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; @@ -58,13 +63,13 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * A {@link FieldMapper} for ip addresses. @@ -262,43 +267,99 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { @Override public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); - InetAddress[] addresses = new InetAddress[values.size()]; - int i = 0; - for (Object value : values) { - InetAddress address; - if (value instanceof InetAddress) { - address = (InetAddress) value; - } else { - if (value instanceof BytesRef) { - value = ((BytesRef) value).utf8ToString(); + Tuple, List> ipsMasks = splitIpsAndMasks(values); + List combiner = new ArrayList<>(); + convertIps(ipsMasks.v1(), combiner); + convertMasks(ipsMasks.v2(), context, combiner); + if (combiner.size() == 1) { + return combiner.get(0); + } + return new ConstantScoreQuery(union(combiner)); + } + + private Query union(List combiner) { + BooleanQuery.Builder bqb = new BooleanQuery.Builder(); + for (Query q : combiner) { + bqb.add(q, BooleanClause.Occur.SHOULD); + } + return bqb.build(); + } + + private void convertIps(List inetAddresses, List sink) { + if (!inetAddresses.isEmpty() && (isSearchable() || hasDocValues())) { + Query pointsQuery = null; + if (isSearchable()) { + pointsQuery = inetAddresses.size() == 1 + ? InetAddressPoint.newExactQuery(name(), inetAddresses.iterator().next()) + : InetAddressPoint.newSetQuery(name(), inetAddresses.toArray(new InetAddress[0])); + } + Query dvQuery = null; + if (hasDocValues()) { + List set = new ArrayList<>(inetAddresses.size()); + for (final InetAddress address : inetAddresses) { + set.add(new BytesRef(InetAddressPoint.encode(address))); } - if (value.toString().contains("/")) { - // the `terms` query contains some prefix queries, so we cannot create a set query - // and need to fall back to a disjunction of `term` queries - return super.termsQuery(values, context); + if (set.size() == 1) { + dvQuery = SortedSetDocValuesField.newSlowExactQuery(name(), set.iterator().next()); + } else { + dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), set); } - address = InetAddresses.forString(value.toString()); } - addresses[i++] = address; - } - Query dvQuery = null; - if (hasDocValues()) { - List bytesRefs = Arrays.stream(addresses) - .distinct() - .map(InetAddressPoint::encode) - .map(BytesRef::new) - .collect(Collectors.toList()); - dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs); + final Query out; + if (isSearchable() && hasDocValues()) { + out = new IndexOrDocValuesQuery(pointsQuery, dvQuery); + } else { + out = isSearchable() ? pointsQuery : dvQuery; + } + sink.add(out); } - Query pointQuery = null; - if (isSearchable()) { - pointQuery = InetAddressPoint.newSetQuery(name(), addresses); + } + + private void convertMasks(List masks, QueryShardContext context, List sink) { + if (!masks.isEmpty() && (isSearchable() || hasDocValues())) { + MultiIpRangeQueryBuilder multiRange = null; + for (String mask : masks) { + final Tuple cidr = InetAddresses.parseCidr(mask); + PointRangeQuery query = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + if (isSearchable()) { // even there is DV we don't go with it, since we can't guess clauses limit + if (multiRange == null) { + multiRange = new MultiIpRangeQueryBuilder(name()); + } + multiRange.add(query.getLowerPoint(), query.getUpperPoint()); + } else { // it may hit clauses limit sooner or later + Query dvRange = SortedSetDocValuesField.newSlowRangeQuery( + name(), + new BytesRef(query.getLowerPoint()), + new BytesRef(query.getUpperPoint()), + true, + true + ); + sink.add(dvRange); + } + } + // never IndexOrDocValuesQuery() since we can't guess clauses limit + if (multiRange != null) { + sink.add(multiRange.build()); + } } - if (isSearchable() && hasDocValues()) { - return new IndexOrDocValuesQuery(pointQuery, dvQuery); - } else { - return isSearchable() ? pointQuery : dvQuery; + } + + private static Tuple, List> splitIpsAndMasks(List values) { + List concreteIPs = new ArrayList<>(); + List masks = new ArrayList<>(); + for (final Object value : values) { + if (value instanceof InetAddress) { + concreteIPs.add((InetAddress) value); + } else { + final String strVal = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + if (strVal.contains("/")) { + masks.add(strVal); + } else { + concreteIPs.add(InetAddresses.forString(strVal)); + } + } } + return Tuple.tuple(concreteIPs, masks); } @Override @@ -445,6 +506,30 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } } + /** + * Union over IP address ranges + */ + public static class MultiIpRangeQueryBuilder extends MultiRangeQuery.Builder { + public MultiIpRangeQueryBuilder(String field) { + super(field, InetAddressPoint.BYTES, 1); + } + + public MultiIpRangeQueryBuilder add(InetAddress lower, InetAddress upper) { + add(new MultiRangeQuery.RangeClause(InetAddressPoint.encode(lower), InetAddressPoint.encode(upper))); + return this; + } + + @Override + public MultiRangeQuery build() { + return new MultiRangeQuery(field, numDims, bytesPerDim, clauses) { + @Override + protected String toString(int dimension, byte[] value) { + return NetworkAddress.format(InetAddressPoint.decode(value)); + } + }; + } + } + private final boolean indexed; private final boolean hasDocValues; private final boolean stored; diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index a5403ef81481f..243164c5fe8fb 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -205,14 +205,12 @@ public void testTermsQuery() { ); // if the list includes a prefix query we fallback to a bool query - assertEquals( - new ConstantScoreQuery( - new BooleanQuery.Builder().add(ft.termQuery("::42", null), Occur.SHOULD) - .add(ft.termQuery("::2/16", null), Occur.SHOULD) - .build() - ), - ft.termsQuery(Arrays.asList("::42", "::2/16"), null) - ); + Query actual = ft.termsQuery(Arrays.asList("::42", "::2/16"), null); + assertTrue(actual instanceof ConstantScoreQuery); + assertTrue(((ConstantScoreQuery) actual).getQuery() instanceof BooleanQuery); + BooleanQuery bq = (BooleanQuery) ((ConstantScoreQuery) actual).getQuery(); + assertEquals(2, bq.clauses().size()); + assertTrue(bq.clauses().stream().allMatch(c -> c.getOccur() == Occur.SHOULD)); } public void testDvOnlyTermsQuery() { @@ -238,6 +236,14 @@ public void testDvOnlyTermsQuery() { ); } + public void testDvVsPoint() { + MappedFieldType indexOnly = new IpFieldMapper.IpFieldType("field", true, false, false, null, Collections.emptyMap()); + MappedFieldType dvOnly = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + MappedFieldType indexDv = new IpFieldMapper.IpFieldType("field", true, false, true, null, Collections.emptyMap()); + assertEquals("ignore DV", indexOnly.termsQuery(List.of("::2/16"), null), indexDv.termsQuery(List.of("::2/16"), null)); + assertEquals(dvOnly.termQuery("::2/16", null), dvOnly.termsQuery(List.of("::2/16"), null)); + } + public void testRangeQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); Query query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE); From 6d3fd372954e0ebaae6ae41d4420d940c4a1d9c8 Mon Sep 17 00:00:00 2001 From: Rishabh Maurya Date: Fri, 29 Nov 2024 09:45:44 -0800 Subject: [PATCH 350/357] Library changes for Apache Arrow integration (#16691) * Library changes for arrow integration Signed-off-by: Rishabh Maurya * Bump guava 32->33 Signed-off-by: Rishabh Maurya * add support for onCancel and Cancellable for BatchedJob in lib:arrow module Signed-off-by: Rishabh Maurya * address PR comments Signed-off-by: Rishabh Maurya * Move StreamTicket to an interface Signed-off-by: Rishabh Maurya * remove jackson dependencies Signed-off-by: Rishabh Maurya * make sl4j runtime only Signed-off-by: Rishabh Maurya * introduce factory for stream ticket Signed-off-by: Rishabh Maurya * Address PR comments Signed-off-by: Rishabh Maurya --------- Signed-off-by: Rishabh Maurya --- gradle/libs.versions.toml | 6 +- libs/arrow-spi/build.gradle | 88 + .../licenses/arrow-format-17.0.0.jar.sha1 | 1 + .../licenses/arrow-format-LICENSE.txt | 2261 +++++++++++++++++ .../licenses/arrow-format-NOTICE.txt | 84 + .../arrow-memory-core-17.0.0.jar.sha1 | 1 + .../licenses/arrow-memory-core-LICENSE.txt | 2261 +++++++++++++++++ .../licenses/arrow-memory-core-NOTICE.txt | 84 + .../arrow-memory-netty-17.0.0.jar.sha1 | 1 + .../licenses/arrow-memory-netty-LICENSE.txt | 2261 +++++++++++++++++ .../licenses/arrow-memory-netty-NOTICE.txt | 84 + ...-memory-netty-buffer-patch-17.0.0.jar.sha1 | 1 + ...rrow-memory-netty-buffer-patch-LICENSE.txt | 2261 +++++++++++++++++ ...arrow-memory-netty-buffer-patch-NOTICE.txt | 84 + .../licenses/arrow-vector-17.0.0.jar.sha1 | 1 + .../licenses/arrow-vector-LICENSE.txt | 2261 +++++++++++++++++ .../licenses/arrow-vector-NOTICE.txt | 84 + .../licenses/commons-codec-1.16.1.jar.sha1 | 1 + .../licenses/commons-codec-LICENSE.txt | 202 ++ .../licenses/commons-codec-NOTICE.txt | 5 + .../licenses/flatbuffers-java-2.0.0.jar.sha1 | 1 + .../licenses/flatbuffers-java-LICENSE.txt | 202 ++ .../licenses/flatbuffers-java-NOTICE.txt | 0 .../jackson-annotations-2.17.2.jar.sha1 | 1 + .../licenses/jackson-annotations-LICENSE.txt | 8 + .../licenses/jackson-annotations-NOTICE.txt | 20 + .../licenses/jackson-databind-2.17.2.jar.sha1 | 1 + .../licenses/jackson-databind-LICENSE.txt | 8 + .../licenses/jackson-databind-NOTICE.txt | 20 + .../netty-buffer-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-buffer-LICENSE.txt | 202 ++ .../licenses/netty-buffer-NOTICE.txt | 264 ++ .../netty-common-4.1.115.Final.jar.sha1 | 1 + .../licenses/netty-common-LICENSE.txt | 202 ++ .../licenses/netty-common-NOTICE.txt | 264 ++ .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + libs/arrow-spi/licenses/slf4j-api-LICENSE.txt | 24 + libs/arrow-spi/licenses/slf4j-api-NOTICE.txt | 0 .../opensearch/arrow/spi/StreamManager.java | 57 + .../opensearch/arrow/spi/StreamProducer.java | 157 ++ .../opensearch/arrow/spi/StreamReader.java | 56 + .../opensearch/arrow/spi/StreamTicket.java | 40 + .../arrow/spi/StreamTicketFactory.java | 35 + .../opensearch/arrow/spi/package-info.java | 12 + .../licenses/guava-32.1.1-jre.jar.sha1 | 1 - .../licenses/guava-33.2.1-jre.jar.sha1 | 1 + .../licenses/guava-32.1.1-jre.jar.sha1 | 1 - .../licenses/guava-33.2.1-jre.jar.sha1 | 1 + .../licenses/guava-32.1.1-jre.jar.sha1 | 1 - .../licenses/guava-33.2.1-jre.jar.sha1 | 1 + .../licenses/guava-32.1.1-jre.jar.sha1 | 1 - .../licenses/guava-33.2.1-jre.jar.sha1 | 1 + .../licenses/guava-32.1.1-jre.jar.sha1 | 1 - .../licenses/guava-33.2.1-jre.jar.sha1 | 1 + 54 files changed, 13613 insertions(+), 6 deletions(-) create mode 100644 libs/arrow-spi/build.gradle create mode 100644 libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/arrow-format-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/arrow-format-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/arrow-vector-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/arrow-vector-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 create mode 100644 libs/arrow-spi/licenses/commons-codec-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/commons-codec-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 create mode 100644 libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 libs/arrow-spi/licenses/jackson-databind-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/jackson-databind-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 create mode 100644 libs/arrow-spi/licenses/netty-buffer-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/netty-buffer-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 create mode 100644 libs/arrow-spi/licenses/netty-common-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/netty-common-NOTICE.txt create mode 100644 libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 libs/arrow-spi/licenses/slf4j-api-LICENSE.txt create mode 100644 libs/arrow-spi/licenses/slf4j-api-NOTICE.txt create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicket.java create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicketFactory.java create mode 100644 libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java delete mode 100644 plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1 create mode 100644 plugins/discovery-gce/licenses/guava-33.2.1-jre.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/guava-33.2.1-jre.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 create mode 100644 plugins/repository-azure/licenses/guava-33.2.1-jre.jar.sha1 delete mode 100644 plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/guava-33.2.1-jre.jar.sha1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 55e338efc1cca..4e8aee298345a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -20,7 +20,7 @@ jettison = "1.5.4" woodstox = "6.4.0" kotlin = "1.7.10" antlr4 = "4.13.1" -guava = "32.1.1-jre" +guava = "33.2.1-jre" protobuf = "3.25.5" jakarta_annotation = "1.3.5" google_http_client = "1.44.1" @@ -79,3 +79,7 @@ resteasy = "6.2.4.Final" # opentelemetry dependencies opentelemetry = "1.41.0" opentelemetrysemconv = "1.27.0-alpha" + +# arrow dependencies +arrow = "17.0.0" +flatbuffers = "2.0.0" diff --git a/libs/arrow-spi/build.gradle b/libs/arrow-spi/build.gradle new file mode 100644 index 0000000000000..d14b7e88cfb8c --- /dev/null +++ b/libs/arrow-spi/build.gradle @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +testingConventions.enabled = false +dependencies { + api project(':libs:opensearch-core') + api "org.apache.arrow:arrow-vector:${versions.arrow}" + api "org.apache.arrow:arrow-format:${versions.arrow}" + api "org.apache.arrow:arrow-memory-core:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}" + runtimeOnly "io.netty:netty-buffer:${versions.netty}" + runtimeOnly "io.netty:netty-common:${versions.netty}" + + runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + + implementation "commons-codec:commons-codec:${versions.commonscodec}" +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + // Logging frameworks + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + 'org.slf4j.impl.StaticLoggerBinder', + 'org.slf4j.impl.StaticMDCBinder', + 'org.slf4j.impl.StaticMarkerBinder', + + // Reactor BlockHound + 'reactor.blockhound.BlockHound$Builder', + 'reactor.blockhound.integration.BlockHoundIntegration' + ) + + ignoreViolations( + "io.netty.util.internal.PlatformDependent0", + "io.netty.util.internal.PlatformDependent0\$1", + "io.netty.util.internal.PlatformDependent0\$2", + "io.netty.util.internal.PlatformDependent0\$3", + "io.netty.util.internal.PlatformDependent0\$4", + "io.netty.util.internal.PlatformDependent0\$6", + "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef", + "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef", + "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", + "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields", + "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields", + "io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode", + "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField", + "io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess", + "io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess", + "io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess", + "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField", + "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField", + "org.apache.arrow.memory.ArrowBuf", + "org.apache.arrow.memory.util.ByteFunctionHelpers", + "org.apache.arrow.memory.util.MemoryUtil", + "org.apache.arrow.memory.util.MemoryUtil\$1", + "org.apache.arrow.memory.util.hash.MurmurHasher", + "org.apache.arrow.memory.util.hash.SimpleHasher", + "org.apache.arrow.vector.BaseFixedWidthVector", + "org.apache.arrow.vector.BitVectorHelper", + "org.apache.arrow.vector.Decimal256Vector", + "org.apache.arrow.vector.DecimalVector", + "org.apache.arrow.vector.util.DecimalUtility", + "org.apache.arrow.vector.util.VectorAppender" + ) +} diff --git a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 new file mode 100644 index 0000000000000..34fd4704eac91 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 @@ -0,0 +1 @@ +5d052f20fd1193840eb59818515e710156c364b2 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-format-LICENSE.txt b/libs/arrow-spi/licenses/arrow-format-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-format-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/arrow-spi/licenses/arrow-format-NOTICE.txt b/libs/arrow-spi/licenses/arrow-format-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-format-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 new file mode 100644 index 0000000000000..ea312f4f5e51a --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 @@ -0,0 +1 @@ +51c5287ef5a624656bb38da7684078905b1a88c9 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt b/libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt b/libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 new file mode 100644 index 0000000000000..f77b3d836b77b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 @@ -0,0 +1 @@ +de65a34dfeada4d47b161871fa39fa0a2ab4c39c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt b/libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt b/libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 new file mode 100644 index 0000000000000..b21b4e8cc7d23 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 @@ -0,0 +1 @@ +cdfdaa1bd5135bd869515fc205392ba92dcc1509 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 new file mode 100644 index 0000000000000..8f9fddc882396 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 @@ -0,0 +1 @@ +16685545e4734382c1fcdaf12ac9b0a7d1fc06c0 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-vector-LICENSE.txt b/libs/arrow-spi/licenses/arrow-vector-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-vector-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/libs/arrow-spi/licenses/arrow-vector-NOTICE.txt b/libs/arrow-spi/licenses/arrow-vector-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/libs/arrow-spi/licenses/arrow-vector-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 b/libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 new file mode 100644 index 0000000000000..6b8803089c6d7 --- /dev/null +++ b/libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 @@ -0,0 +1 @@ +47bd4d333fba53406f6c6c51884ddbca435c8862 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/commons-codec-LICENSE.txt b/libs/arrow-spi/licenses/commons-codec-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/libs/arrow-spi/licenses/commons-codec-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/arrow-spi/licenses/commons-codec-NOTICE.txt b/libs/arrow-spi/licenses/commons-codec-NOTICE.txt new file mode 100644 index 0000000000000..e6d6ef34a3e83 --- /dev/null +++ b/libs/arrow-spi/licenses/commons-codec-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Codec +Copyright 2002-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). diff --git a/libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 b/libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 new file mode 100644 index 0000000000000..08a9b7e399e74 --- /dev/null +++ b/libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 @@ -0,0 +1 @@ +19da0c1d9f585d2c402057f993f8dea2ff382837 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt b/libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt b/libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 new file mode 100644 index 0000000000000..411e1d62459fd --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 @@ -0,0 +1 @@ +147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt b/libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt b/libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt new file mode 100644 index 0000000000000..4c976b7b4cc58 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt @@ -0,0 +1,20 @@ +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. diff --git a/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 new file mode 100644 index 0000000000000..f2b4dbdc5decb --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 @@ -0,0 +1 @@ +e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt b/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt new file mode 100644 index 0000000000000..f5f45d26a49d6 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt @@ -0,0 +1,8 @@ +This copy of Jackson JSON processor streaming parser/generator is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivate works. + +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 diff --git a/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt b/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt new file mode 100644 index 0000000000000..4c976b7b4cc58 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt @@ -0,0 +1,20 @@ +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. diff --git a/libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 b/libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..825b91b71601d --- /dev/null +++ b/libs/arrow-spi/licenses/netty-buffer-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +d5daf1030e5c36d198caf7562da2441a97ec0df6 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/netty-buffer-LICENSE.txt b/libs/arrow-spi/licenses/netty-buffer-LICENSE.txt new file mode 100644 index 0000000000000..62589edd12a37 --- /dev/null +++ b/libs/arrow-spi/licenses/netty-buffer-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/arrow-spi/licenses/netty-buffer-NOTICE.txt b/libs/arrow-spi/licenses/netty-buffer-NOTICE.txt new file mode 100644 index 0000000000000..971865b7c1c23 --- /dev/null +++ b/libs/arrow-spi/licenses/netty-buffer-NOTICE.txt @@ -0,0 +1,264 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * https://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * https://www.slf4j.org/ + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * NOTICE: + * license/NOTICE.harmony.txt + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * https://archive.apache.org/dist/harmony/ + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product optionally depends on 'zstd-jni', a zstd-jni Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.zstd-jni.txt (BSD) + * HOMEPAGE: + * https://github.com/luben/zstd-jni + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * https://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jboss-remoting/jboss-marshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * https://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * https://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: + + * LICENSE: + * license/LICENSE.hyper-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/python-hyper/hpack/ + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: + + * LICENSE: + * license/LICENSE.nghttp2-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/nghttp2/nghttp2/ + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + +This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. + + * LICENSE: + * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/takari/maven-wrapper + +This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +This private header is also used by Apple's open source + mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). + + * LICENSE: + * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) + * HOMEPAGE: + * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h + +This product optionally depends on 'Brotli4j', Brotli compression and +decompression for Java., which can be obtained at: + + * LICENSE: + * license/LICENSE.brotli4j.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/hyperxpro/Brotli4j diff --git a/libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 b/libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 new file mode 100644 index 0000000000000..2ff84dc14147d --- /dev/null +++ b/libs/arrow-spi/licenses/netty-common-4.1.115.Final.jar.sha1 @@ -0,0 +1 @@ +9da10a9f72e3f87e181d91b525174007a6fc4f11 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/netty-common-LICENSE.txt b/libs/arrow-spi/licenses/netty-common-LICENSE.txt new file mode 100644 index 0000000000000..62589edd12a37 --- /dev/null +++ b/libs/arrow-spi/licenses/netty-common-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libs/arrow-spi/licenses/netty-common-NOTICE.txt b/libs/arrow-spi/licenses/netty-common-NOTICE.txt new file mode 100644 index 0000000000000..971865b7c1c23 --- /dev/null +++ b/libs/arrow-spi/licenses/netty-common-NOTICE.txt @@ -0,0 +1,264 @@ + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * https://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * https://www.slf4j.org/ + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * NOTICE: + * license/NOTICE.harmony.txt + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * https://archive.apache.org/dist/harmony/ + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product optionally depends on 'zstd-jni', a zstd-jni Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.zstd-jni.txt (BSD) + * HOMEPAGE: + * https://github.com/luben/zstd-jni + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * https://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jboss-remoting/jboss-marshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * https://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * https://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: + + * LICENSE: + * license/LICENSE.hyper-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/python-hyper/hpack/ + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: + + * LICENSE: + * license/LICENSE.nghttp2-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/nghttp2/nghttp2/ + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + +This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. + + * LICENSE: + * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/takari/maven-wrapper + +This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +This private header is also used by Apple's open source + mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). + + * LICENSE: + * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) + * HOMEPAGE: + * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h + +This product optionally depends on 'Brotli4j', Brotli compression and +decompression for Java., which can be obtained at: + + * LICENSE: + * license/LICENSE.brotli4j.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/hyperxpro/Brotli4j diff --git a/libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 b/libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/slf4j-api-LICENSE.txt b/libs/arrow-spi/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..1a3d053237bec --- /dev/null +++ b/libs/arrow-spi/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,24 @@ +Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland) +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + diff --git a/libs/arrow-spi/licenses/slf4j-api-NOTICE.txt b/libs/arrow-spi/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java new file mode 100644 index 0000000000000..cdb83f032356a --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.tasks.TaskId; + +/** + * Interface for managing Arrow data streams between producers and consumers. + * StreamManager handles the registration of producers, stream access control via tickets, + * and coordinates the lazy initialization of Arrow resources. It ensures proper lifecycle + * management of streaming resources across distributed nodes. + * + *

    Implementation of this interface should ensure thread-safety and proper resource cleanup. + * The manager uses tickets as a mechanism to securely transfer stream access rights between + * producers and consumers.

    + */ +@ExperimentalApi +public interface StreamManager extends AutoCloseable { + + /** + * Registers a stream producer and returns a ticket for stream access. + * The registration stores the producer reference but delays Arrow resource + * initialization until the first consumer connects. + * + * @param producer The StreamProducer that will generate Arrow data + * @param parentTaskId The TaskId that identifies the parent operation creating this stream + * @return A StreamTicket that can be used to access the stream + * @throws IllegalArgumentException if producer is null or parentTaskId is invalid + */ + StreamTicket registerStream(StreamProducer producer, TaskId parentTaskId); + + /** + * Creates a stream reader for consuming Arrow data using a valid ticket. + * This method may trigger lazy initialization of Arrow resources if this is + * the first access to the stream. + * + * @param ticket The StreamTicket obtained from registerStream + * @return A StreamReader for consuming the Arrow data + * @throws IllegalArgumentException if the ticket is invalid + * @throws IllegalStateException if the stream has been cancelled or closed + */ + StreamReader getStreamReader(StreamTicket ticket); + + /** + * Gets the StreamTicketFactory instance associated with this StreamManager. + * + * @return the StreamTicketFactory instance + */ + StreamTicketFactory getStreamTicketFactory(); +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java new file mode 100644 index 0000000000000..c5cd6f16adfdd --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.tasks.TaskId; + +import java.io.Closeable; + +/** + * Represents a producer of Arrow streams. The producer first needs to define the job by implementing this interface and + * then register the job with the {@link StreamManager#registerStream(StreamProducer, TaskId)}, which will return {@link StreamTicket} + * which can be distributed to the consumer. The consumer can then use the ticket to retrieve the stream using + * {@link StreamManager#getStreamReader(StreamTicket)} and then consume the stream using {@link StreamReader}. + *

    + * BatchedJob supports streaming of intermediate results, allowing consumers to begin processing data before the entire + * result set is generated. This is particularly useful for memory-intensive operations or when dealing with large datasets + * that shouldn't be held entirely in memory. + *

    + * Example usage: + * 

    {@code
+ * public class QueryStreamProducer implements StreamProducer {
+ *     private final SearchRequest searchRequest;
+ *     private static final int BATCH_SIZE = 1000;
+ *
+ *     @Override
+ *     public VectorSchemaRoot createRoot(BufferAllocator allocator) {
+ *         List fields = Arrays.asList(
+ *             Field.nullable("id", FieldType.valueOf(MinorType.VARCHAR)),
+ *             Field.nullable("score", FieldType.valueOf(MinorType.FLOAT8))
+ *         );
+ *         return VectorSchemaRoot.create(new Schema(fields), allocator);
+ *     }
+ *
+ *     @Override
+ *     public BatchedJob createJob(BufferAllocator allocator) {
+ *         return new BatchedJob() {
+ *             @Override
+ *             public void run(VectorSchemaRoot root, FlushSignal flushSignal) {
+ *                 SearchResponse response = client.search(searchRequest);
+ *                 int currentBatch = 0;
+ *
+ *                 VarCharVector idVector = (VarCharVector) root.getVector("id");
+ *                 Float8Vector scoreVector = (Float8Vector) root.getVector("score");
+ *
+ *                 for (SearchHit hit : response.getHits()) {
+ *                     idVector.setSafe(currentBatch, hit.getId().getBytes());
+ *                     scoreVector.setSafe(currentBatch, hit.getScore());
+ *                     currentBatch++;
+ *
+ *                     if (currentBatch >= BATCH_SIZE) {
+ *                         root.setRowCount(currentBatch);
+ *                         flushSignal.awaitConsumption(1000);
+ *                         currentBatch = 0;
+ *                     }
+ *                 }
+ *             }
+ *         };
+ *     }
+ * }
+ *
+ * // Usage:
+ * StreamProducer producer = new QueryStreamProducer(searchRequest);
+ * StreamTicket ticket = streamManager.registerStream(producer, taskId);
+ * }
    + * + * @see StreamManager + * @see StreamTicket + * @see StreamReader + */ +@ExperimentalApi +public interface StreamProducer extends Closeable { + + /** + * Creates a VectorSchemaRoot that defines the schema for this stream. This schema will be used + * for all batches produced by this stream. + * + * @param allocator The allocator to use for creating vectors + * @return A new VectorSchemaRoot instance + */ + VectorSchemaRoot createRoot(BufferAllocator allocator); + + /** + * Creates a job that will produce the stream data in batches. The job will populate + * the VectorSchemaRoot and use FlushSignal to coordinate with consumers. + * + * @param allocator The allocator to use for any additional memory allocations + * @return A new BatchedJob instance + */ + BatchedJob createJob(BufferAllocator allocator); + + /** + * Provides an estimate of the total number of rows that will be produced. + * + * @return Estimated number of rows, or -1 if unknown + */ + int estimatedRowCount(); + + /** + * Task action name + * @return action name + */ + String getAction(); + + /** + * BatchedJob interface for producing stream data in batches. + */ + interface BatchedJob { + + /** + * Executes the batch processing job. Implementations should populate the root with data + * and use flushSignal to coordinate with consumers when each batch is ready. + * + * @param root The VectorSchemaRoot to populate with data + * @param flushSignal Signal to coordinate with consumers + */ + void run(VectorSchemaRoot root, FlushSignal flushSignal); + + /** + * Called to signal producer when the job is canceled. + * This method is used to clean up resources or cancel ongoing operations. + * This maybe called from a different thread than the one used for run(). It might be possible that run() + * thread is busy when onCancel() is called and wakes up later. In such cases, ensure that run() terminates early + * and should clean up resources. + */ + void onCancel(); + + /** + * Producers can set isCancelled flag to true to indicate that the job is canceled. + * This will ensure the stream is closed and no more data is produced from next Batch onwards. + * + * @return true if the job is canceled, false otherwise + */ + boolean isCancelled(); + } + + /** + * Functional interface for managing stream consumption signals. + */ + @FunctionalInterface + interface FlushSignal { + /** + * Blocks until the current batch has been consumed or timeout occurs. + * + * @param timeout Maximum milliseconds to wait + */ + void awaitConsumption(int timeout); + } +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java new file mode 100644 index 0000000000000..b258652988b96 --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.Closeable; + +/** + * StreamReader is a stateful iterator that can be used to read data from a stream. + * It is used to read data from a stream in batches. The iterator will return a + * VectorSchemaRoot that contains the data for the current batch. The iterator will + * return true if there is more data to read, false if the stream is exhausted. + * Example usage: + * 
    {@code
+ * // producer
+ * StreamProducer producer = new QueryStreamProducer(searchRequest);
+ * StreamTicket ticket = streamManager.registerStream(producer, taskId);
+ *
+ * // consumer
+ * StreamReader iterator = streamManager.getStreamReader(ticket);
+ * try (VectorSchemaRoot root = iterator.getRoot()) {
+ *     while (iterator.next()) {
+ *         VarCharVector idVector = (VarCharVector)root.getVector("id");
+ *         Float8Vector scoreVector = (Float8Vector) root.getVector("score");
+ *     }
+ * }
+ * }
    + * + * @see StreamProducer + */ +@ExperimentalApi +public interface StreamReader extends Closeable { + + /** + * Blocking request to load next batch into root. + * + * @return true if more data was found, false if the stream is exhausted + */ + boolean next(); + + /** + * Returns the VectorSchemaRoot associated with this iterator. + * The content of this root is updated with each successful call to next(). + * + * @return the VectorSchemaRoot + */ + VectorSchemaRoot getRoot(); +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicket.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicket.java new file mode 100644 index 0000000000000..e86e317273b49 --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicket.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * A ticket that uniquely identifies a stream. This ticket is created when a producer registers + * a stream with {@link StreamManager} and can be used by consumers to retrieve the stream using + * {@link StreamManager#getStreamReader(StreamTicket)}. + */ +@ExperimentalApi +public interface StreamTicket { + /** + * Returns the ticketId associated with this stream ticket. + * + * @return the ticketId string + */ + String getTicketId(); + + /** + * Returns the nodeId associated with this stream ticket. + * + * @return the nodeId string + */ + String getNodeId(); + + /** + * Serializes this ticket into a Base64 encoded byte array. + * + * @return Base64 encoded byte array containing the ticket information + */ + byte[] toBytes(); +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicketFactory.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicketFactory.java new file mode 100644 index 0000000000000..6e042b7f16c8d --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamTicketFactory.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.spi; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Factory interface for creating and managing StreamTicket instances. + * This factory provides methods to create and deserialize StreamTickets, + * ensuring consistent ticket creation. + */ +@ExperimentalApi +public interface StreamTicketFactory { + /** + * Creates a new StreamTicket + * + * @return A new StreamTicket instance + */ + StreamTicket newTicket(); + + /** + * Deserializes a StreamTicket from its byte representation. + * + * @param bytes The byte array containing the serialized ticket data + * @return A StreamTicket instance reconstructed from the byte array + * @throws IllegalArgumentException if bytes is null or invalid + */ + StreamTicket fromBytes(byte[] bytes); +} diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java new file mode 100644 index 0000000000000..d075ecaa764bb --- /dev/null +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Contains Apache Arrow related classes and Stream generic interfaces + */ +package org.opensearch.arrow.spi; diff --git a/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1 deleted file mode 100644 index 0d791b5d3f55b..0000000000000 --- a/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/discovery-gce/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/discovery-gce/licenses/guava-33.2.1-jre.jar.sha1 new file mode 100644 index 0000000000000..27d5304e326df --- /dev/null +++ b/plugins/discovery-gce/licenses/guava-33.2.1-jre.jar.sha1 @@ -0,0 +1 @@ +818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 deleted file mode 100644 index 0d791b5d3f55b..0000000000000 --- a/plugins/ingest-attachment/licenses/guava-32.1.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/ingest-attachment/licenses/guava-33.2.1-jre.jar.sha1 new file mode 100644 index 0000000000000..27d5304e326df --- /dev/null +++ b/plugins/ingest-attachment/licenses/guava-33.2.1-jre.jar.sha1 @@ -0,0 +1 @@ +818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 deleted file mode 100644 index 0d791b5d3f55b..0000000000000 --- a/plugins/repository-azure/licenses/guava-32.1.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/repository-azure/licenses/guava-33.2.1-jre.jar.sha1 new file mode 100644 index 0000000000000..27d5304e326df --- /dev/null +++ b/plugins/repository-azure/licenses/guava-33.2.1-jre.jar.sha1 @@ -0,0 +1 @@ +818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 deleted file mode 100644 index 0d791b5d3f55b..0000000000000 --- a/plugins/repository-gcs/licenses/guava-32.1.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 new file mode 100644 index 0000000000000..27d5304e326df --- /dev/null +++ b/plugins/repository-gcs/licenses/guava-33.2.1-jre.jar.sha1 @@ -0,0 +1 @@ +818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 deleted file mode 100644 index 0d791b5d3f55b..0000000000000 --- a/plugins/repository-hdfs/licenses/guava-32.1.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad575652d84153075dd41ec6177ccb15251262b2 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/guava-33.2.1-jre.jar.sha1 b/plugins/repository-hdfs/licenses/guava-33.2.1-jre.jar.sha1 new file mode 100644 index 0000000000000..27d5304e326df --- /dev/null +++ b/plugins/repository-hdfs/licenses/guava-33.2.1-jre.jar.sha1 @@ -0,0 +1 @@ +818e780da2c66c63bbb6480fef1f3855eeafa3e4 \ No newline at end of file From b75f27aa347f26aa528e291db81809491c79142f Mon Sep 17 00:00:00 2001 From: kkewwei Date: Sat, 30 Nov 2024 03:24:38 +0800 Subject: [PATCH 351/357] Fix Flaky Test SearchWithRandomExceptionsIT.testRandomExceptions (#16737) Signed-off-by: kkewwei Signed-off-by: kkewwei --- .../search/basic/SearchWithRandomExceptionsIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java index 614ec2ebd634a..e4a8ac3f705d4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWithRandomExceptionsIT.java @@ -193,6 +193,11 @@ public void testRandomExceptions() throws IOException, InterruptedException, Exe logger.info("expected SearchPhaseException: [{}]", ex.getMessage()); } } + + // as the index refresh may fail, so the translog in the index will be not flushed, + // and `TranslogWriter.buffer` is not null, which causes arrays not been released, + // so we need to close the index to release the arrays. + cluster().wipeIndices("test"); } public static final String EXCEPTION_TOP_LEVEL_RATIO_KEY = "index.engine.exception.ratio.top"; From d2a1477ac452db8bbb31a8988c51dec62b8ea23f Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Mon, 2 Dec 2024 20:44:45 +0800 Subject: [PATCH 352/357] Deprecate performing update operation with default pipeline or final pipeline (#16712) * Deprecate performing update operation with default pipeline or final pipeline Signed-off-by: Gao Binlong * Modify the warning message Signed-off-by: Gao Binlong * Modify changelog Signed-off-by: Gao Binlong * Fix test issue Signed-off-by: Gao Binlong --------- Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../rest-api-spec/test/ingest/75_update.yml | 102 ++++++++++++++++++ .../action/update/TransportUpdateAction.java | 14 ++- 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index c81586548d210..57c2f97405fd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391) ### Deprecated +- Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712)) ### Removed diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml new file mode 100644 index 0000000000000..a66b6293110cf --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml @@ -0,0 +1,102 @@ +setup: + - do: + ingest.put_pipeline: + id: "pipeline1" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field1", + "value": "value1" + } + } + ] + } + - do: + indices.create: + index: test_1 + body: + settings: + index.default_pipeline: "pipeline1" + - do: + indices.create: + index: test_2 + body: + settings: + index.final_pipeline: "pipeline1" +--- +teardown: + - do: + ingest.delete_pipeline: + id: "pipeline1" + ignore: 404 + + - do: + indices.delete: + index: test_1 + - do: + indices.delete: + index: test_2 +--- +"update operation with predefined default or final pipeline returns warning header": + - skip: + version: " - 2.99.99" + reason: "this change is added in 3.0.0" + features: allowed_warnings + - do: + index: + index: test_1 + id: 1 + body: { foo: bar } + + - match: { _seq_no: 0 } + - match: { _version: 1 } + - match: { _primary_term: 1 } + - match: { result: created } + + - do: + allowed_warnings: + - "the index [test_1] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0" + update: + index: test_1 + id: 1 + _source: true + body: + doc: { foo: bar1 } + + - match: { _seq_no: 1 } + - match: { _primary_term: 1 } + - match: { _version: 2 } + - match: { result: updated } + - match: { get._source.foo: bar1 } + - match: { get._source.field1: value1 } + + - do: + index: + index: test_2 + id: 1 + body: { foo: bar } + + - match: { _seq_no: 0 } + - match: { _version: 1 } + - match: { _primary_term: 1 } + - match: { result: created } + + - do: + allowed_warnings: + - "the index [test_2] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0" + update: + index: test_2 + id: 1 + _source: true + body: + doc: { foo: bar1 } + + - match: { _seq_no: 1 } + - match: { _primary_term: 1 } + - match: { _version: 2 } + - match: { result: updated } + - match: { get._source.foo: bar1 } + - match: { get._source.field1: value1 } diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index 819112eb497f6..52378142ae1dd 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -57,6 +57,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Inject; +import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; @@ -67,6 +69,7 @@ import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; +import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats; @@ -90,7 +93,7 @@ * @opensearch.internal */ public class TransportUpdateAction extends TransportInstanceSingleOperationAction { - + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransportUpdateAction.class); private final AutoCreateIndex autoCreateIndex; private final UpdateHelper updateHelper; private final IndicesService indicesService; @@ -276,6 +279,15 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< IndexRequest indexRequest = result.action(); // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request final BytesReference indexSourceBytes = indexRequest.source(); + final Settings indexSettings = indexService.getIndexSettings().getSettings(); + if (IndexSettings.DEFAULT_PIPELINE.exists(indexSettings) || IndexSettings.FINAL_PIPELINE.exists(indexSettings)) { + deprecationLogger.deprecate( + "update_operation_with_ingest_pipeline", + "the index [" + + indexRequest.index() + + "] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0" + ); + } client.bulk(toSingleItemBulkRequest(indexRequest), wrapBulkResponse(ActionListener.wrap(response -> { UpdateResponse update = new UpdateResponse( response.getShardInfo(), From ac456430d6159a21a8899a95bc64208f6f4c9eb5 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 2 Dec 2024 20:24:29 -0500 Subject: [PATCH 353/357] Update 75_update.yml (#16759) Signed-off-by: Andriy Redko --- .../resources/rest-api-spec/test/ingest/75_update.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml index a66b6293110cf..b3567edd2545e 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml @@ -42,8 +42,8 @@ teardown: --- "update operation with predefined default or final pipeline returns warning header": - skip: - version: " - 2.99.99" - reason: "this change is added in 3.0.0" + version: " - 2.18.99" + reason: "this change is added in 2.19.0" features: allowed_warnings - do: index: From d6157d6e67a6aad2d308e59ba16d9f6501a5ef13 Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Tue, 3 Dec 2024 20:53:09 +0800 Subject: [PATCH 354/357] Fix allowed_warnings takes no effect for the yaml test in 75_update.yml (#16764) Signed-off-by: Gao Binlong --- .../resources/rest-api-spec/test/ingest/75_update.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml index b3567edd2545e..d223631c09324 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml @@ -44,7 +44,7 @@ teardown: - skip: version: " - 2.18.99" reason: "this change is added in 2.19.0" - features: allowed_warnings + features: [warnings, headers] - do: index: index: test_1 @@ -57,7 +57,8 @@ teardown: - match: { result: created } - do: - allowed_warnings: + headers: { "X-Opaque-Id": "default_pipeline_request" } + warnings: - "the index [test_1] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0" update: index: test_1 @@ -85,7 +86,8 @@ teardown: - match: { result: created } - do: - allowed_warnings: + headers: { "X-Opaque-Id": "final_pipeline_request" } + warnings: - "the index [test_2] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0" update: index: test_2 From 80ba41f3b459f58af40df5553283e2456a640a38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 08:21:26 -0500 Subject: [PATCH 355/357] Bump ch.qos.logback:logback-classic from 1.2.13 to 1.5.12 in /test/fixtures/hdfs-fixture (#16716) * Bump ch.qos.logback:logback-classic in /test/fixtures/hdfs-fixture Bumps [ch.qos.logback:logback-classic](https://github.com/qos-ch/logback) from 1.2.13 to 1.5.12. - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.13...v_1.5.12) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-classic dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c2f97405fd2..c6cc316219269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) - Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667)) - Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718)) +- Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716)) ### Changed - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391) diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 55e27c7d6169b..f531a3c6ade5a 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -75,7 +75,7 @@ dependencies { api "org.apache.commons:commons-text:1.12.0" api "commons-net:commons-net:3.11.1" api "ch.qos.logback:logback-core:1.5.12" - api "ch.qos.logback:logback-classic:1.2.13" + api "ch.qos.logback:logback-classic:1.5.12" api "org.jboss.xnio:xnio-nio:3.8.16.Final" api 'org.jline:jline:3.27.1' api 'org.apache.commons:commons-configuration2:2.11.0' From 5b05dcba6c8f2672c6113ebdad607148b570dab1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 3 Dec 2024 13:11:33 -0500 Subject: [PATCH 356/357] Update Jackson to 2.18.2 (#16758) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 | 1 - client/sniffer/licenses/jackson-core-2.18.2.jar.sha1 | 1 + .../upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + .../upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - .../upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + gradle/libs.versions.toml | 4 ++-- libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + libs/core/licenses/jackson-core-2.17.2.jar.sha1 | 1 - libs/core/licenses/jackson-core-2.18.2.jar.sha1 | 1 + libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 | 1 - libs/x-content/licenses/jackson-core-2.18.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-smile-2.17.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-smile-2.18.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 | 1 + .../ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + .../ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - .../ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + .../crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + .../discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - .../discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + .../licenses/jackson-databind-2.17.2.jar.sha1 | 1 - .../licenses/jackson-databind-2.18.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-xml-2.17.2.jar.sha1 | 1 - .../licenses/jackson-dataformat-xml-2.18.2.jar.sha1 | 1 + .../licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 | 1 - .../licenses/jackson-datatype-jsr310-2.18.2.jar.sha1 | 1 + .../licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 | 1 - .../licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.17.2.jar.sha1 | 1 - .../licenses/jackson-annotations-2.18.2.jar.sha1 | 1 + .../repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 | 1 - .../repository-s3/licenses/jackson-databind-2.18.2.jar.sha1 | 1 + server/licenses/jackson-core-2.17.2.jar.sha1 | 1 - server/licenses/jackson-core-2.18.2.jar.sha1 | 1 + server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 | 1 + server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 | 1 + server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 | 1 - server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 | 1 + 56 files changed, 30 insertions(+), 29 deletions(-) delete mode 100644 client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 create mode 100644 client/sniffer/licenses/jackson-core-2.18.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 libs/core/licenses/jackson-core-2.17.2.jar.sha1 create mode 100644 libs/core/licenses/jackson-core-2.18.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-core-2.18.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.18.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.18.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.18.2.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.18.2.jar.sha1 delete mode 100644 server/licenses/jackson-core-2.17.2.jar.sha1 create mode 100644 server/licenses/jackson-core-2.18.2.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 delete mode 100644 server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 create mode 100644 server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index c6cc316219269..2564b78354697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665)) - Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667)) - Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718)) +- Bump `jackson` from 2.17.2 to 2.18.2 ([#16733](https://github.com/opensearch-project/OpenSearch/pull/16733)) - Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716)) ### Changed diff --git a/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 b/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 deleted file mode 100644 index e15f2340980bc..0000000000000 --- a/client/sniffer/licenses/jackson-core-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -969a35cb35c86512acbadcdbbbfb044c877db814 \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-2.18.2.jar.sha1 b/client/sniffer/licenses/jackson-core-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..96350c9307ae7 --- /dev/null +++ b/client/sniffer/licenses/jackson-core-2.18.2.jar.sha1 @@ -0,0 +1 @@ +fb64ccac5c27dca8819418eb4e443a9f496d9ee7 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4e8aee298345a..2658d2aa8b561 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,8 +8,8 @@ bundled_jdk = "23.0.1+11" # optional dependencies spatial4j = "0.7" jts = "1.15.0" -jackson = "2.17.2" -jackson_databind = "2.17.2" +jackson = "2.18.2" +jackson_databind = "2.18.2" snakeyaml = "2.1" icu4j = "75.1" supercsv = "2.4.0" diff --git a/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/libs/arrow-spi/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/libs/arrow-spi/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 b/libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/libs/core/licenses/jackson-core-2.17.2.jar.sha1 b/libs/core/licenses/jackson-core-2.17.2.jar.sha1 deleted file mode 100644 index e15f2340980bc..0000000000000 --- a/libs/core/licenses/jackson-core-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -969a35cb35c86512acbadcdbbbfb044c877db814 \ No newline at end of file diff --git a/libs/core/licenses/jackson-core-2.18.2.jar.sha1 b/libs/core/licenses/jackson-core-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..96350c9307ae7 --- /dev/null +++ b/libs/core/licenses/jackson-core-2.18.2.jar.sha1 @@ -0,0 +1 @@ +fb64ccac5c27dca8819418eb4e443a9f496d9ee7 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 deleted file mode 100644 index e15f2340980bc..0000000000000 --- a/libs/x-content/licenses/jackson-core-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -969a35cb35c86512acbadcdbbbfb044c877db814 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.18.2.jar.sha1 b/libs/x-content/licenses/jackson-core-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..96350c9307ae7 --- /dev/null +++ b/libs/x-content/licenses/jackson-core-2.18.2.jar.sha1 @@ -0,0 +1 @@ +fb64ccac5c27dca8819418eb4e443a9f496d9ee7 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 deleted file mode 100644 index 069e088413ef1..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57fa7c1b5104bbc4599278d13933a937ee058e68 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..8b946b98ddbf9 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 @@ -0,0 +1 @@ +d4870757eff0344130f60e3ddb882b2336640f73 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 deleted file mode 100644 index 28d8c8382aed3..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20e956b9b6f67138edd39fab7a506ded19638bcb \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..9fbdb9b3a2506 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 @@ -0,0 +1 @@ +44caf62d743bb5e5876e95efba5a55a1cab1b0db \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 deleted file mode 100644 index f3e25b7eb253c..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78d2c73dbec62044d7cf3b544b2e0d24a1a093b0 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..9dac9ee8e1e72 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 @@ -0,0 +1 @@ +d000e13505d1cf564371516fa3d5b8769a779dc9 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/plugins/crypto-kms/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/plugins/crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/plugins/crypto-kms/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/plugins/repository-azure/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/plugins/repository-azure/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 deleted file mode 100644 index f9c31c168926d..0000000000000 --- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad58f5bd089e743ac6e5999b2d1e3cf8515cea9a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.18.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..61ee41aa8adf4 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.18.2.jar.sha1 @@ -0,0 +1 @@ +069cb3b7bd34b3f7842cc4a6fd717981433bf73e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 deleted file mode 100644 index a61bf643d69e6..0000000000000 --- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -267b85e9ba2892a37be6d80aa9ca1438a0d8c210 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.18.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..7b9ab1d1e08d1 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.18.2.jar.sha1 @@ -0,0 +1 @@ +7b6ff96adf421f4c6edbd694e797dd8fe434510a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 deleted file mode 100644 index d9d7975146c22..0000000000000 --- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2978b818ef2f2b2738b387c143624eab611d917 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..b98599718965b --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +048c96032e5a428739e28ff04973717c032df598 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1 deleted file mode 100644 index 411e1d62459fd..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147b7b9412ffff24339f8aba080b292448e08698 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..a06e1d5f28425 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.18.2.jar.sha1 @@ -0,0 +1 @@ +985d77751ebc7fce5db115a986bc9aa82f973f4a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 deleted file mode 100644 index f2b4dbdc5decb..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6deb029e5901e027c129341fac39e515066b68c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..eedbfff66c705 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.18.2.jar.sha1 @@ -0,0 +1 @@ +deef8697b92141fb6caf7aa86966cff4eec9b04f \ No newline at end of file diff --git a/server/licenses/jackson-core-2.17.2.jar.sha1 b/server/licenses/jackson-core-2.17.2.jar.sha1 deleted file mode 100644 index e15f2340980bc..0000000000000 --- a/server/licenses/jackson-core-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -969a35cb35c86512acbadcdbbbfb044c877db814 \ No newline at end of file diff --git a/server/licenses/jackson-core-2.18.2.jar.sha1 b/server/licenses/jackson-core-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..96350c9307ae7 --- /dev/null +++ b/server/licenses/jackson-core-2.18.2.jar.sha1 @@ -0,0 +1 @@ +fb64ccac5c27dca8819418eb4e443a9f496d9ee7 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 deleted file mode 100644 index 069e088413ef1..0000000000000 --- a/server/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57fa7c1b5104bbc4599278d13933a937ee058e68 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 b/server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..8b946b98ddbf9 --- /dev/null +++ b/server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1 @@ -0,0 +1 @@ +d4870757eff0344130f60e3ddb882b2336640f73 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 deleted file mode 100644 index 28d8c8382aed3..0000000000000 --- a/server/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20e956b9b6f67138edd39fab7a506ded19638bcb \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 b/server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..9fbdb9b3a2506 --- /dev/null +++ b/server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1 @@ -0,0 +1 @@ +44caf62d743bb5e5876e95efba5a55a1cab1b0db \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 deleted file mode 100644 index f3e25b7eb253c..0000000000000 --- a/server/licenses/jackson-dataformat-yaml-2.17.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78d2c73dbec62044d7cf3b544b2e0d24a1a093b0 \ No newline at end of file diff --git a/server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 b/server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 new file mode 100644 index 0000000000000..9dac9ee8e1e72 --- /dev/null +++ b/server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1 @@ -0,0 +1 @@ +d000e13505d1cf564371516fa3d5b8769a779dc9 \ No newline at end of file From b1bf72f26e2681e4dbe726bc9605209675f6ab38 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 3 Dec 2024 11:59:46 -0800 Subject: [PATCH 357/357] Bound the size of cache in deprecation logger (#16724) The current implementation of the map used to de-duplicate deprecation log messages can grow without bound. This adds a simple fixed limit to the data structure tracking existing loggers. Once the limit is breached new loggers will no longer log deprecation warnings. I also added a check to skip the tracking if the deprecation logger is disabled. Signed-off-by: Andrew Ross --- CHANGELOG.md | 1 + .../common/logging/DeprecatedMessage.java | 22 +++++++++++++++---- .../common/logging/DeprecationLogger.java | 9 +++++--- .../logging/DeprecationLoggerTests.java | 18 +++++++++++++++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2564b78354697..5c89dd7b53374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628)) - Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670)) - [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560)) +- Bound the size of cache in deprecation logger ([16702](https://github.com/opensearch-project/OpenSearch/issues/16702)) ### Security diff --git a/server/src/main/java/org/opensearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/opensearch/common/logging/DeprecatedMessage.java index 25c1ba9675600..b906752e74b31 100644 --- a/server/src/main/java/org/opensearch/common/logging/DeprecatedMessage.java +++ b/server/src/main/java/org/opensearch/common/logging/DeprecatedMessage.java @@ -47,12 +47,17 @@ */ public class DeprecatedMessage extends OpenSearchLogMessage { public static final String X_OPAQUE_ID_FIELD_NAME = "x-opaque-id"; - private static final Set keys = ConcurrentHashMap.newKeySet(); + + // Arbitrary maximum size, should be much larger than unique number of + // loggers, but small relative to heap size. + static final int MAX_DEDUPE_CACHE_ENTRIES = 16_384; + + private static final Set keyDedupeCache = ConcurrentHashMap.newKeySet(); private final String keyWithXOpaqueId; public DeprecatedMessage(String key, String xOpaqueId, String messagePattern, Object... args) { super(fieldMap(key, xOpaqueId), messagePattern, args); - this.keyWithXOpaqueId = new StringBuilder().append(key).append(xOpaqueId).toString(); + this.keyWithXOpaqueId = key + xOpaqueId; } /** @@ -62,7 +67,7 @@ public DeprecatedMessage(String key, String xOpaqueId, String messagePattern, Ob * Otherwise, a warning can be logged by some test and the upcoming test can be impacted by it. */ public static void resetDeprecatedMessageForTests() { - keys.clear(); + keyDedupeCache.clear(); } private static Map fieldMap(String key, String xOpaqueId) { @@ -77,6 +82,15 @@ private static Map fieldMap(String key, String xOpaqueId) { } public boolean isAlreadyLogged() { - return !keys.add(keyWithXOpaqueId); + if (keyDedupeCache.contains(keyWithXOpaqueId)) { + return true; + } + if (keyDedupeCache.size() >= MAX_DEDUPE_CACHE_ENTRIES) { + // Stop logging if max size is breached to avoid performance problems from + // excessive logging. The historical logs will be full of deprecation warnings + // at this point anyway. + return true; + } + return !keyDedupeCache.add(keyWithXOpaqueId); } } diff --git a/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java b/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java index d4dbb953ffe12..7a1911f6a83c8 100644 --- a/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java +++ b/server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java @@ -116,9 +116,12 @@ public DeprecationLoggerBuilder deprecate(final String key, final String msg, fi public class DeprecationLoggerBuilder { public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[] params) { - DeprecatedMessage deprecationMessage = new DeprecatedMessage(key, HeaderWarning.getXOpaqueId(), msg, params); - if (!deprecationMessage.isAlreadyLogged()) { - logger.log(DEPRECATION, deprecationMessage); + // Check if the logger is enabled to skip the overhead of deduplicating messages if the logger is disabled + if (logger.isEnabled(DEPRECATION)) { + DeprecatedMessage deprecationMessage = new DeprecatedMessage(key, HeaderWarning.getXOpaqueId(), msg, params); + if (!deprecationMessage.isAlreadyLogged()) { + logger.log(DEPRECATION, deprecationMessage); + } } return this; } diff --git a/server/src/test/java/org/opensearch/common/logging/DeprecationLoggerTests.java b/server/src/test/java/org/opensearch/common/logging/DeprecationLoggerTests.java index 96ee7831c20ed..98fa1fc4022fe 100644 --- a/server/src/test/java/org/opensearch/common/logging/DeprecationLoggerTests.java +++ b/server/src/test/java/org/opensearch/common/logging/DeprecationLoggerTests.java @@ -69,4 +69,22 @@ public void testDuplicateLogMessages() { // assert that only unique warnings are logged assertWarnings("Deprecated message 1", "Deprecated message 2", "Deprecated message 3"); } + + public void testMaximumSizeOfCache() { + final int maxEntries = DeprecatedMessage.MAX_DEDUPE_CACHE_ENTRIES; + // Fill up the cache, asserting every message is new + for (int i = 0; i < maxEntries; i++) { + DeprecatedMessage message = new DeprecatedMessage("key-" + i, "message-" + i, ""); + assertFalse(message.toString(), message.isAlreadyLogged()); + } + // Do the same thing except assert every message has been seen + for (int i = 0; i < maxEntries; i++) { + DeprecatedMessage message = new DeprecatedMessage("key-" + i, "message-" + i, ""); + assertTrue(message.toString(), message.isAlreadyLogged()); + } + // Add one more new entry, asserting it will forever been seen as already logged (cache is full) + DeprecatedMessage message = new DeprecatedMessage("key-new", "message-new", ""); + assertTrue(message.toString(), message.isAlreadyLogged()); + assertTrue(message.toString(), message.isAlreadyLogged()); + } }